From b69a6ad6aea778c8d7a08f80966b69d6107248a7 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Mon, 7 Sep 2026 22:08:13 +0200 Subject: [PATCH 1/6] feat(credentials): add opt-in workspace credential sources Bind supported agent channel/search collections by source UID, project only into owned namespaces, refresh and revoke consumers without silent legacy fallback, and preserve the existing direct path when unset. Add CLI lifecycle, schema, documentation, and a disposable Kind consumer gate. Independent review, live CI evidence, and genuine audit sign-offs remain pending. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/commands/add.ts | 51 +- cli/src/commands/credentials-source.test.ts | 77 ++ cli/src/commands/credentials.ts | 47 +- cli/src/lib/credential-source-io.ts | 155 ++++ cli/src/lib/credential-source.test.ts | 335 +++++++++ cli/src/lib/credential-source.ts | 227 ++++++ cli/src/testing/credential-source.test.ts | 74 ++ controller/src/crd.rs | 5 + controller/src/credential_source.rs | 129 ++++ controller/src/main.rs | 1 + controller/src/reconciler/agent_env.rs | 64 ++ .../credential_source_projection.rs | 211 ++++++ .../credential_source_test_server.rs | 344 +++++++++ .../src/reconciler/credential_source_tests.rs | 701 ++++++++++++++++++ .../reconciler/credential_source_workloads.rs | 117 +++ .../src/reconciler/credential_sources.rs | 509 +++++++++++++ controller/src/reconciler/mod.rs | 113 +-- deploy/helm/kars/templates/crd.yaml | 20 + docs/how-to/credential-sources.md | 182 +++++ docs/how-to/helm-installation.md | 4 + .../2026-09-07-credential-sources.md | 96 +++ tests/e2e/credential-sources.sh | 173 +++++ tests/e2e/run.sh | 2 + 23 files changed, 3524 insertions(+), 113 deletions(-) create mode 100644 cli/src/commands/credentials-source.test.ts create mode 100644 cli/src/lib/credential-source-io.ts create mode 100644 cli/src/lib/credential-source.test.ts create mode 100644 cli/src/lib/credential-source.ts create mode 100644 cli/src/testing/credential-source.test.ts create mode 100644 controller/src/credential_source.rs create mode 100644 controller/src/reconciler/agent_env.rs create mode 100644 controller/src/reconciler/credential_source_projection.rs create mode 100644 controller/src/reconciler/credential_source_test_server.rs create mode 100644 controller/src/reconciler/credential_source_tests.rs create mode 100644 controller/src/reconciler/credential_source_workloads.rs create mode 100644 controller/src/reconciler/credential_sources.rs create mode 100644 docs/how-to/credential-sources.md create mode 100644 docs/security-audits/2026-09-07-credential-sources.md create mode 100644 tests/e2e/credential-sources.sh diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index 55c5a01e..77f0efe1 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -7,6 +7,8 @@ import ora from "ora"; import { loadContext, resolveSecret } from "../config.js"; import { assertRuntimeWired, buildRuntimeBlock, flagToKind } from "../runtime.js"; import { CLAIM, prepareCredentialNamespace } from "../lib/namespace-ownership.js"; +import { applySourceSandbox, prepareCredentialSource, updatesFromFlags, waitForCredentialSource, type SourceRef } from "../lib/credential-source.js"; +import { FLAG_ENV, SOURCE_KEYS, targetName } from "../lib/credential-source-io.js"; import { buildInferencePolicy, buildToolPolicy, @@ -20,6 +22,8 @@ export function addCommand(): Command { cmd .description("Add a new sandboxed agent to an existing kars cluster") .argument("", "Name for the new sandbox agent") + .option("--namespace ", "Workspace for the Sandbox and policy CRs", "kars-system") + .option("--credential-source", "Use a pinned workspace credential source instead of runtime-namespace credentials") // ── Core (all runtimes) ──────────────────────────────────────────── .option("--runtime ", "Runtime kind: openclaw | openai-agents | microsoft-agent-framework | langgraph | anthropic | pydantic-ai | hermes | byo", "openclaw") @@ -87,6 +91,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. `) .action(async (name: string, options) => { const { execa } = await import("execa"); + targetName(name, options.namespace); const runtimeKind = flagToKind(options.runtime); assertRuntimeWired(runtimeKind); @@ -110,7 +115,11 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. ["--image", options.image], ]; if (runtimeKind !== "OpenClaw") { - const used = openClawOnlyFlags.filter(([, v]) => v !== undefined && v !== "" && v !== false).map(([f]) => f); + const sourceFlags = options.credentialSource ? new Set(Object.entries(FLAG_ENV) + .filter(([, env]) => (SOURCE_KEYS as readonly string[]).includes(env)) + .map(([flag]) => `--${flag.replace(/[A-Z]/g, letter => `-${letter.toLowerCase()}`)}`)) : new Set(); + const used = openClawOnlyFlags.filter(([flag, value]) => !sourceFlags.has(flag) + && value !== undefined && value !== "" && value !== false).map(([flag]) => flag); if (used.length > 0) { console.error(chalk.red(`\n Error: ${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with --runtime openclaw.`)); console.error(chalk.dim(` Channels, skills, and plugin API keys are OpenClaw-specific entrypoint features.`)); @@ -146,7 +155,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. kind: "KarsSandbox", metadata: { name, - namespace: "kars-system", + namespace: options.namespace, }, spec: { runtime: runtimeBlock, @@ -292,7 +301,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. // S13: build companion same-namespace policy CRs (sibling to KarsSandbox). const inferencePolicy = buildInferencePolicy({ sandboxName: name, - namespace: "kars-system", + namespace: options.namespace, model: options.model, provider: "azure-ai-foundry", contentSafety: true, @@ -305,7 +314,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. const toolPolicy = options.governance ? buildToolPolicy({ sandboxName: name, - namespace: "kars-system", + namespace: options.namespace, profile: options.policyProfile || "default", }) : undefined; @@ -316,6 +325,11 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. const yaml = JSON.stringify(bundle, null, 2); if (options.dryRun) { + if (options.credentialSource) { + console.log(`Plan: create/update the opted-in source in ${options.namespace}, then bind its API-assigned UID when submitting ${name}.`); + console.log("No runnable source-bound manifest is emitted before a real UID exists. No credential values are printed."); + return; + } console.log(chalk.bold("\nKarsSandbox manifest (dry-run):\n")); console.log(yaml); console.log(chalk.dim("\nApply with: kubectl apply -f ")); @@ -447,10 +461,21 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. const allSecrets = { ...channelEnvSecrets, ...pluginSecrets, + ...(options.credentialSource ? updatesFromFlags(options) : {}), }; - if (Object.keys(allSecrets).length > 0) { + let sourceReference: SourceRef | undefined; + if (options.credentialSource) { + const prepared = await prepareCredentialSource(execa, name, options.namespace, allSecrets); + const policies = bundle.filter(item => item !== sandbox); + await execa("kubectl", ["apply", "-f", "-"], { + input: JSON.stringify({ apiVersion: "v1", kind: "List", items: policies }), + stdio: ["pipe", "pipe", "pipe"], + }); + await applySourceSandbox(execa, sandbox, prepared); + sourceReference = prepared.reference; + } else if (Object.keys(allSecrets).length > 0) { spinner.text = "Creating credential secret..."; - const namespaceUid = await prepareCredentialNamespace(execa, name, "kars-system"); + const namespaceUid = await prepareCredentialNamespace(execa, name, options.namespace); const metadata = sandbox.metadata as Record; metadata.annotations = { ...(metadata.annotations as Record | undefined), @@ -479,10 +504,14 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. kind: "List", items: bundle, }; - await execa("kubectl", ["apply", "-f", "-"], { - input: JSON.stringify(bundleManifest), - stdio: ["pipe", "pipe", "pipe"], - }); + if (sourceReference) { + await waitForCredentialSource(execa, name, options.namespace, sourceReference); + } else { + await execa("kubectl", ["apply", "-f", "-"], { + input: JSON.stringify(bundleManifest), + stdio: ["pipe", "pipe", "pipe"], + }); + } // The controller auto-mounts -credentials secret via envFrom (optional: true). // If the secret exists, env vars are injected into the sandbox container at startup. @@ -602,7 +631,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. if (options.skills) { console.log(chalk.dim(` Skills: ${options.skills}`)); } - console.log(chalk.dim(` Status: kubectl get karssandbox ${name} -n kars-system`)); + console.log(chalk.dim(` Status: kubectl get karssandbox ${name} -n ${options.namespace}`)); console.log(chalk.dim(` Connect: kars connect ${name}`)); console.log(chalk.dim(` Remove: kars destroy ${name}\n`)); diff --git a/cli/src/commands/credentials-source.test.ts b/cli/src/commands/credentials-source.test.ts new file mode 100644 index 00000000..fa203ed1 --- /dev/null +++ b/cli/src/commands/credentials-source.test.ts @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { credentialsCommand } from "./credentials.js"; +import { addCommand } from "./add.js"; + +const mocks = vi.hoisted(() => ({ + source: vi.fn(), direct: vi.fn(), execute: vi.fn(), + spinner: { start: vi.fn(), succeed: vi.fn(), fail: vi.fn(), warn: vi.fn() }, +})); +vi.mock("execa", () => ({ execa: mocks.execute })); +vi.mock("ora", () => ({ default: () => mocks.spinner })); +vi.mock("../config.js", async importOriginal => ({ + ...await importOriginal(), + resolveSecret: (_value: string | undefined) => undefined, + loadContext: () => undefined, +})); +vi.mock("../lib/credential-source.js", async importOriginal => ({ + ...await importOriginal(), + updateCredentialSource: mocks.source, updateDirectCredentials: mocks.direct, +})); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.spinner.start.mockReturnValue(mocks.spinner); + mocks.execute.mockResolvedValue({ stdout: "" }); + mocks.direct.mockResolvedValue(undefined); + mocks.source.mockResolvedValue({ kind: "source", reference: { name: "source", uid: "source-uid" } }); + vi.spyOn(console, "log").mockImplementation(() => {}); +}); +afterEach(() => vi.restoreAllMocks()); + +describe("credentials command source integration", () => { + it("routes opt-in flags to the workspace and never prints source values or restarts directly", async () => { + await credentialsCommand().parseAsync([ + "update", "demo", "--namespace", "workspace-a", "--use-source", "--telegram-token", "SENSITIVE-INPUT", + ], { from: "user" }); + expect(mocks.source).toHaveBeenCalledWith(mocks.execute, "demo", "workspace-a", { + updates: { TELEGRAM_BOT_TOKEN: "SENSITIVE-INPUT" }, remove: [], + useSource: true, disableSource: undefined, restart: true, + }); + expect(mocks.direct).not.toHaveBeenCalled(); + expect(mocks.execute).not.toHaveBeenCalled(); + expect(vi.mocked(console.log).mock.calls.flat().join(" ")).not.toContain("SENSITIVE-INPUT"); + }); + + it("supports remote removal and explicit source disable using existing credentials update", async () => { + await credentialsCommand().parseAsync(["update", "demo", "--remove", "telegram-token"], { from: "user" }); + expect(mocks.source.mock.calls[0][3].remove).toEqual(["TELEGRAM_BOT_TOKEN"]); + await credentialsCommand().parseAsync(["update", "demo", "--disable-source"], { from: "user" }); + expect(mocks.source.mock.calls[1][3].disableSource).toBe(true); + }); + + it("keeps legacy flags and --no-restart working on the direct path", async () => { + mocks.source.mockResolvedValue(undefined); + await credentialsCommand().parseAsync([ + "update", "demo", "--openai-api-key", "legacy-provider", "--no-restart", + ], { from: "user" }); + expect(mocks.direct).toHaveBeenCalledWith(mocks.execute, "demo", { OPENAI_API_KEY: "legacy-provider" }, []); + expect(mocks.execute).not.toHaveBeenCalled(); + }); +}); + +describe("source-mode add flags", () => { + it.each(["openclaw", "openai-agents", "microsoft-agent-framework", "langgraph", "anthropic", "pydantic-ai", "hermes"])( + "permits agent credential environment flags for %s without publishing a fake UID", async runtime => { + await addCommand().parseAsync([ + "demo", "--runtime", runtime, "--credential-source", "--telegram-token", "SENSITIVE-INPUT", "--dry-run", + ], { from: "user" }); + expect(mocks.execute).not.toHaveBeenCalled(); + const output = vi.mocked(console.log).mock.calls.flat().join(" "); + expect(output).toContain("No runnable source-bound manifest"); + expect(output).not.toContain("SENSITIVE-INPUT"); + }, + ); +}); diff --git a/cli/src/commands/credentials.ts b/cli/src/commands/credentials.ts index 4b689b29..3fe97775 100644 --- a/cli/src/commands/credentials.ts +++ b/cli/src/commands/credentials.ts @@ -3,6 +3,7 @@ import { Command } from "commander"; import chalk from "chalk"; +import { removedKeys, updateCredentialSource, updateDirectCredentials } from "../lib/credential-source.js"; import { banner, section } from "../stepper.js"; import { promptAndSaveCredentials, SECRETS_FILE, @@ -233,8 +234,12 @@ export function credentialsCommand(): Command { // Subcommand: update credentials for a running AKS sandbox const update = new Command("update"); update - .description("Update credentials for a running AKS sandbox (updates secret + restarts pod)") + .description("Update sandbox credentials (direct Secret or pinned workspace source)") .argument("", "Sandbox name") + .option("--namespace ", "Workspace containing the Sandbox CR", "kars-system") + .option("--use-source", "Opt in to a UID-pinned workspace source (migrates direct credentials once)") + .option("--disable-source", "Explicitly restore the unchanged legacy credential collection") + .option("--remove ", "Remove keys (credential flag names or environment keys, comma separated)") .option("--telegram-token ", "New Telegram bot token") .option("--telegram-allow-from ", "Telegram allowed user IDs (comma-separated)") .option("--slack-token ", "New Slack bot token") @@ -269,40 +274,29 @@ export function credentialsCommand(): Command { if (options[flag]) updates[env] = options[flag]; } - if (Object.keys(updates).length === 0) { + const remove = removedKeys(options.remove); + if (Object.keys(updates).length === 0 && remove.length === 0 && !options.useSource && !options.disableSource) { console.error(chalk.red(" No credentials specified. Use --telegram-token, --brave-api-key, etc.")); process.exit(1); } const namespace = `kars-${name}`; - const secretName = `${name}-credentials`; const spinner = ora(`Updating credentials for '${name}'...`).start(); try { - // Read existing secret (if any) and merge with new values - let existing: Record = {}; - try { - const { stdout } = await execa("kubectl", [ - "get", "secret", secretName, "-n", namespace, - "-o", "jsonpath={.data}", - ], { stdio: "pipe" }); - if (stdout && stdout !== "{}") { - const data = JSON.parse(stdout); - for (const [k, v] of Object.entries(data)) { - existing[k] = Buffer.from(v as string, "base64").toString(); - } - } - } catch { /* secret doesn't exist yet */ } - - const merged = { ...existing, ...updates }; - - // Create/replace the secret - const secretArgs = ["create", "secret", "generic", secretName, "-n", namespace, "--dry-run=client", "-o", "yaml"]; - for (const [env, val] of Object.entries(merged)) { - secretArgs.push(`--from-literal=${env}=${val}`); + const source = await updateCredentialSource(execa, name, options.namespace, { + updates, remove, useSource: options.useSource, disableSource: options.disableSource, + restart: options.restart, + }); + if (source) { + spinner.succeed(source.staged + ? `Source staged; bind it with kars add ${name} --credential-source` + : source.reference ? "Credential source reconciled; controller owns runtime refresh" + : "Source disabled; controller will restore direct credentials"); + if (source.reference) console.log(chalk.dim(` Source: ${options.namespace}/${source.reference.name} (UID ${source.reference.uid})`)); + return; } - const { stdout: yaml } = await execa("kubectl", secretArgs, { stdio: "pipe" }); - await execa("kubectl", ["apply", "-f", "-"], { input: yaml, stdio: ["pipe", "pipe", "pipe"] }); + await updateDirectCredentials(execa, name, updates, remove); spinner.succeed("Secret updated"); @@ -310,6 +304,7 @@ export function credentialsCommand(): Command { for (const [env, val] of Object.entries(updates)) { console.log(chalk.dim(` ${env} = ••••${val.slice(-4)}`)); } + for (const key of remove) console.log(chalk.dim(` Removed ${key}`)); // Restart pod unless --no-restart if (options.restart !== false) { diff --git a/cli/src/lib/credential-source-io.ts b/cli/src/lib/credential-source-io.ts new file mode 100644 index 00000000..af4b5c0d --- /dev/null +++ b/cli/src/lib/credential-source-io.ts @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Execute } from "./deployment-target.js"; + +export interface Metadata { + name: string; namespace?: string; uid: string; resourceVersion: string; + annotations?: Record; + ownerReferences?: Array<{ apiVersion: string; kind: string; name: string; uid: string; controller?: boolean; blockOwnerDeletion?: boolean }>; + deletionTimestamp?: string; +} +export interface SourceRef { name: string; uid: string } +export interface Sandbox { + metadata: Metadata; + spec: { credentialsRef?: SourceRef | null; [key: string]: unknown }; + status?: { conditions?: Array<{ type: string; status: string; reason: string; message: string }> }; +} +export interface Secret { + apiVersion?: string; kind?: string; type?: string; + metadata: Metadata; data?: Record; +} +export const SOURCE_KEYS = [ + "TELEGRAM_BOT_TOKEN", "TELEGRAM_ALLOW_FROM", "SLACK_BOT_TOKEN", "DISCORD_BOT_TOKEN", + "WHATSAPP_ENABLED", "BRAVE_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "FIRECRAWL_API_KEY", + "PERPLEXITY_API_KEY", +] as const; +export const SOURCE = { + prefix: "kars-credential-source-", + purpose: "kars.azure.com/credential-purpose", + target: "kars.azure.com/credential-target", + intent: "kars.azure.com/credential-binding-intent", + sandboxUid: "kars.azure.com/credential-sandbox-uid", + workspace: "kars.azure.com/credential-workspace", + namespaceUid: "kars.azure.com/credential-namespace-uid", +} as const; +export const FLAG_ENV: Record = { + telegramToken: "TELEGRAM_BOT_TOKEN", telegramAllowFrom: "TELEGRAM_ALLOW_FROM", + slackToken: "SLACK_BOT_TOKEN", discordToken: "DISCORD_BOT_TOKEN", + braveApiKey: "BRAVE_API_KEY", tavilyApiKey: "TAVILY_API_KEY", exaApiKey: "EXA_API_KEY", + firecrawlApiKey: "FIRECRAWL_API_KEY", perplexityApiKey: "PERPLEXITY_API_KEY", openaiApiKey: "OPENAI_API_KEY", +}; + +export function targetName(name: string, workspace: string): string { + for (const value of [name, workspace]) { + if (value.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(value)) { + throw new Error("Credential target/workspace must be Kubernetes DNS labels"); + } + } + return `${SOURCE.prefix}${name}`; +} + +export function identity(meta: Metadata): void { + if (!meta?.name || !meta.uid || !meta.resourceVersion || meta.deletionTimestamp) { + throw new Error("Credential API identity is incomplete or terminating"); + } +} + +export async function run( + execute: Execute, stage: string, args: string[], input?: object, +): Promise { + try { + const result = await execute("kubectl", args, { + stdio: "pipe", ...(input ? { input: JSON.stringify(input) } : {}), + }); + return String(result.stdout); + } catch (error) { + // execa/admission errors can contain the entire Secret request. Never + // attach the original error, stdout, stderr, or command input as a cause. + const code = (error as { exitCode?: unknown }).exitCode; + throw new Error(`${stage} failed (kubectl exit ${typeof code === "number" ? code : "unknown"})`); + } +} + +export function parse(text: string): T { + try { return JSON.parse(text) as T; } + catch { throw new Error("Credential API returned invalid JSON"); } +} + +export async function get( + execute: Execute, kind: string, name: string, namespace?: string, +): Promise { + const text = await run(execute, "Read credential resource", [ + "get", kind, name, ...(namespace ? ["-n", namespace] : []), "--ignore-not-found", "-o", "json", + ]); + if (!text.trim()) return undefined; + const value = parse(text); + identity(value.metadata); + if (value.metadata.name !== name || (namespace && value.metadata.namespace !== namespace)) { + throw new Error("Credential API returned a different resource identity"); + } + return value; +} + +export function decode(secret?: Secret): Record { + const values: Record = Object.create(null); + for (const [key, value] of Object.entries(secret?.data ?? {})) { + try { + if (typeof value !== "string") throw new Error(); + const buffer = Buffer.from(value, "base64"); + if (buffer.toString("base64") !== value) throw new Error(); + values[key] = new TextDecoder("utf-8", { fatal: true }).decode(buffer); + } catch { throw new Error("Credential Secret has invalid encoded environment data"); } + } + return values; +} + +export function validateValues(values: Record): void { + let size = 0; + for (const [key, value] of Object.entries(values)) { + if (!(SOURCE_KEYS as readonly string[]).includes(key) || value.includes("\0")) { + throw new Error("Source mode accepts only supported agent channel/search credentials, not provider/control-plane or arbitrary environment keys"); + } + size += Buffer.byteLength(value); + } + if (size > 131_072) throw new Error("Credential source exceeds 128 KiB"); +} + +export function validateSource(secret: Secret, name: string, workspace: string, sandbox?: Sandbox): void { + identity(secret.metadata); + const meta = secret.metadata; + const annotations = meta.annotations ?? {}; + if (meta.name !== targetName(name, workspace) || meta.namespace !== workspace || secret.type !== "Opaque" + || annotations[SOURCE.purpose] !== "agent-source-v1" || annotations[SOURCE.target] !== name + || annotations[SOURCE.workspace] !== workspace || annotations[SOURCE.intent] !== "explicit-reference-v1") { + throw new Error("Existing source has incompatible purpose, type, or target; no takeover"); + } + const refs = meta.ownerReferences ?? []; + if (refs.length && (!sandbox || refs.length !== 1 || refs[0].apiVersion !== "kars.azure.com/v1alpha1" + || refs[0].kind !== "KarsSandbox" || refs[0].name !== name || refs[0].uid !== sandbox.metadata.uid + || refs[0].controller !== true || refs[0].blockOwnerDeletion !== false)) { + throw new Error("Source belongs to a different Sandbox incarnation"); + } + if (annotations[SOURCE.sandboxUid] !== undefined && annotations[SOURCE.sandboxUid] !== sandbox?.metadata.uid) { + throw new Error("Source Sandbox UID binding differs; no automatic reuse"); + } + if (annotations[SOURCE.namespaceUid] !== undefined + && annotations[SOURCE.namespaceUid] !== sandbox?.metadata.annotations?.["kars.azure.com/namespace-uid"]) { + throw new Error("Source runtime namespace binding differs"); + } +} + +export function updatesFromFlags(options: Record): Record { + return Object.fromEntries(Object.entries(FLAG_ENV) + .filter(([flag]) => typeof options[flag] === "string" && options[flag] !== "") + .map(([flag, env]) => [env, options[flag] as string])); +} + +export function removedKeys(value?: string): string[] { + return (value ?? "").split(",").map(key => key.trim()).filter(Boolean).map(key => { + const flag = key.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase()); + const env = FLAG_ENV[flag] ?? key; + if (!/^[A-Z_][A-Z0-9_]*$/.test(env)) throw new Error("Credential removal requires a flag name or environment key"); + return env; + }); +} diff --git a/cli/src/lib/credential-source.test.ts b/cli/src/lib/credential-source.test.ts new file mode 100644 index 00000000..37840447 --- /dev/null +++ b/cli/src/lib/credential-source.test.ts @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync } from "node:fs"; +import { describe, expect, it, vi } from "vitest"; +import type { Execute } from "./deployment-target.js"; +import { + applySourceSandbox, prepareCredentialSource, updateCredentialSource, + updateDirectCredentials, verifyCredentialReference, waitForCredentialSource, +} from "./credential-source.js"; +import { + SOURCE, SOURCE_KEYS, decode, removedKeys, targetName, validateValues, + type Sandbox, type Secret, +} from "./credential-source-io.js"; +import { CLAIM } from "./namespace-ownership.js"; + +const encode = (values: Record) => Object.fromEntries( + Object.entries(values).map(([key, value]) => [key, Buffer.from(value).toString("base64")]), +); +const reference = { name: "kars-credential-source-demo", uid: "source-uid" }; +const opts = (extra = {}) => ({ updates: {}, remove: [], ...extra }); + +function sandbox(): Sandbox { + return { + metadata: { name: "demo", namespace: "kars-system", uid: "sandbox-uid", resourceVersion: "10", + annotations: { [CLAIM.namespaceUid]: "namespace-uid" } }, + spec: { runtime: { kind: "OpenClaw", openclaw: {} } }, + }; +} + +function source(): Secret { + return { + apiVersion: "v1", kind: "Secret", type: "Opaque", + metadata: { + name: reference.name, namespace: "kars-system", uid: reference.uid, resourceVersion: "20", + annotations: { + [SOURCE.purpose]: "agent-source-v1", [SOURCE.target]: "demo", + [SOURCE.workspace]: "kars-system", [SOURCE.intent]: "explicit-reference-v1", + "customer.example/note": "preserve", + }, + }, + data: encode({ TELEGRAM_BOT_TOKEN: "initial-source" }), + }; +} + +function cluster() { + const state = { + sandbox: sandbox() as Sandbox | undefined, + source: undefined as Secret | undefined, + namespace: { + metadata: { + name: "kars-demo", uid: "namespace-uid", resourceVersion: "30", + annotations: { + [CLAIM.version]: "v1", [CLAIM.name]: "demo", [CLAIM.namespace]: "kars-system", + [CLAIM.uid]: "sandbox-uid", + }, + }, + } as { metadata: { name: string; uid: string; resourceVersion: string; annotations: Record } } | undefined, + legacy: { + apiVersion: "v1", kind: "Secret", type: "Opaque", + metadata: { name: "demo-credentials", namespace: "kars-demo", uid: "legacy-uid", resourceVersion: "40", + annotations: { "customer.example/keep": "yes" } }, + data: encode({ TELEGRAM_BOT_TOKEN: "legacy-token", SLACK_BOT_TOKEN: "keep-slack" }), + } as Secret | undefined, + failRead: false, failWrite: false, pruneRef: false, acknowledge: true, revision: 100, + recreateSourceOnPatch: false, conflictCreate: false, + }; + const run = vi.fn(async (command: string, args: readonly string[], options?: { input?: string }) => { + expect(command).toBe("kubectl"); + expect(args.some(arg => arg.startsWith("--from-literal"))).toBe(false); + if (args[0] === "get") { + if (state.failRead) throw Object.assign(new Error("forbidden: private-token-value"), { exitCode: 1 }); + let object: unknown; + if (args[1] === "karssandbox") { + if (state.acknowledge && state.sandbox?.spec.credentialsRef && state.source) { + state.sandbox.status = { conditions: [{ + type: "CredentialsReady", status: "True", reason: "Projected", + message: JSON.stringify({ sourceUid: state.source.metadata.uid, sourceVersion: state.source.metadata.resourceVersion }), + }] }; + } + object = state.sandbox; + } else if (args[1] === "namespace") object = state.namespace; + else object = args[2] === reference.name ? state.source : state.legacy; + if (!object) return { stdout: "" }; + if (args.includes("jsonpath={.metadata}")) object = (object as Secret).metadata; + return { stdout: JSON.stringify(object) }; + } + if (state.failWrite) throw Object.assign(new Error(`admission echoed ${options?.input}`), { exitCode: 1 }); + const body = JSON.parse(options!.input!); + if (args[0] === "create") { + if (state.conflictCreate) throw Object.assign(new Error("AlreadyExists"), { exitCode: 1 }); + if (body.kind === "Secret") { + expect(state.source).toBeUndefined(); + body.metadata.uid = reference.uid; + body.metadata.resourceVersion = String(++state.revision); + state.source = body; + } else { + expect(body.kind).toBe("KarsSandbox"); + expect(state.sandbox).toBeUndefined(); + body.metadata.uid = "sandbox-created"; + body.metadata.resourceVersion = String(++state.revision); + if (state.pruneRef) delete body.spec.credentialsRef; + state.sandbox = body; + } + return { stdout: JSON.stringify(body) }; + } + expect(args[0]).toBe("patch"); + expect(args).toContain("--patch-file=/dev/stdin"); + const object = args[1] === "karssandbox" ? state.sandbox + : args[2] === reference.name ? state.source : state.legacy; + expect(object).toBeDefined(); + if (state.recreateSourceOnPatch && object === state.source) { + object!.metadata.uid = "replacement"; + object!.metadata.resourceVersion = "replacement-version"; + } + if (object!.metadata.uid !== body.metadata.uid || object!.metadata.resourceVersion !== body.metadata.resourceVersion) { + throw Object.assign(new Error("Conflict"), { exitCode: 1 }); + } + if (args[1] === "karssandbox") { + if (state.pruneRef) delete body.spec.credentialsRef; + if (body.spec.credentialsRef === null) delete state.sandbox!.spec.credentialsRef; + else state.sandbox!.spec = { ...state.sandbox!.spec, ...body.spec }; + } else { + const secret = object as Secret; + secret.data ??= {}; + for (const [key, value] of Object.entries(body.data)) { + if (value === null) delete secret.data[key]; + else secret.data[key] = value as string; + } + } + object!.metadata.resourceVersion = String(++state.revision); + return { stdout: JSON.stringify(object) }; + }); + return { state, run, execute: run as unknown as Execute }; +} + +describe("credential source contract", () => { + it("matches the core and existing handoff agent-key allowlist", () => { + const rust = readFileSync(new URL("../../../controller/src/credential_source.rs", import.meta.url), "utf8"); + const block = rust.slice(rust.indexOf("pub const AGENT_KEYS"), rust.indexOf("pub fn source_name")); + expect([...block.matchAll(/"([A-Z][A-Z0-9_]+)"/g)].map(match => match[1])).toEqual([...SOURCE_KEYS]); + expect(SOURCE_KEYS).not.toContain("OPENAI_API_KEY"); + }); + + it.each(["OPENAI_API_KEY", "AZURE_CLIENT_SECRET", "AGT_KEY", "NODE_OPTIONS", "PATH"])( + "rejects reserved/provider key %s without echoing values", key => { + expect(() => validateValues({ [key]: "private-value" })).toThrow("supported agent"); + try { validateValues({ [key]: "private-value" }); } + catch (error) { expect(String(error)).not.toContain("private-value"); } + }, + ); + + it("normalizes removal keys and rejects path-shaped targets", () => { + expect(removedKeys("telegram-token,SLACK_BOT_TOKEN")).toEqual(["TELEGRAM_BOT_TOKEN", "SLACK_BOT_TOKEN"]); + expect(() => targetName("../other", "kars-system")).toThrow("DNS"); + expect(() => validateValues({ TELEGRAM_BOT_TOKEN: "a\0b" })).toThrow("environment"); + }); +}); + +describe("real source creation and migration", () => { + it("creates the source before the CR, pins its returned UID, and never writes a runtime namespace", async () => { + const { state, run, execute } = cluster(); + state.sandbox = undefined; state.namespace = undefined; state.legacy = undefined; + const prepared = await prepareCredentialSource(execute, "demo", "kars-system", { TELEGRAM_BOT_TOKEN: "sensitive-token-123" }); + expect(prepared.reference).toEqual(reference); + expect(state.sandbox).toBeUndefined(); + await applySourceSandbox(execute, { + apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", + metadata: { name: "demo", namespace: "kars-system" }, spec: { runtime: { kind: "OpenClaw", openclaw: {} } }, + }, prepared); + expect(state.sandbox).toMatchObject({ spec: { credentialsRef: reference } }); + const creates = run.mock.calls.filter(([, args]) => args[0] === "create"); + expect(creates).toHaveLength(2); + expect(JSON.parse(creates[0][2]!.input!).kind).toBe("Secret"); + expect(JSON.parse(creates[1][2]!.input!).kind).toBe("KarsSandbox"); + expect(run.mock.calls.some(([, args]) => args.includes("kars-demo"))).toBe(false); + expect(run.mock.calls.some(([, args]) => args.join(" ").includes("sensitive-token-123"))).toBe(false); + }); + + it("migrates the complete legacy collection once without changing its object or UID", async () => { + const { state, execute, run } = cluster(); + const legacy = structuredClone(state.legacy); + const result = await updateCredentialSource(execute, "demo", "kars-system", opts({ + useSource: true, updates: { TELEGRAM_BOT_TOKEN: "rotated" }, + })); + expect(result?.reference).toEqual(reference); + expect(decode(state.source)).toEqual({ TELEGRAM_BOT_TOKEN: "rotated", SLACK_BOT_TOKEN: "keep-slack" }); + expect(state.legacy).toEqual(legacy); + expect(state.sandbox?.spec.credentialsRef).toEqual(reference); + expect(run.mock.calls.filter(([, args]) => args.includes("kars-demo")).every(([, args]) => args[0] === "get")).toBe(true); + }); + + it("refuses unsafe/incomplete legacy migration before any source write", async () => { + const { state, execute, run } = cluster(); + state.legacy!.data!.OPENAI_API_KEY = Buffer.from("provider-value").toString("base64"); + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ useSource: true }))).rejects.toThrow("supported agent"); + expect(run.mock.calls.every(([, args]) => args[0] === "get")).toBe(true); + expect(state.source).toBeUndefined(); + }); + + it("does not silently attach a source after a racing CR create409 or schema pruning", async () => { + for (const prune of [false, true]) { + const { state, execute } = cluster(); + state.sandbox = undefined; state.namespace = undefined; + const prepared = await prepareCredentialSource(execute, "demo", "kars-system", {}); + state.pruneRef = prune; state.conflictCreate = !prune; + await expect(applySourceSandbox(execute, { + apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", + metadata: { name: "demo", namespace: "kars-system" }, spec: {}, + }, prepared)).rejects.toThrow(); + } + }); +}); + +describe("update, rotation, removal, and path selection", () => { + it("retains the direct path when no reference exists", async () => { + const { state, execute } = cluster(); + expect(await updateCredentialSource(execute, "demo", "kars-system", opts())).toBeUndefined(); + const uid = state.legacy!.metadata.uid; + await updateDirectCredentials(execute, "demo", { OPENAI_API_KEY: "legacy-supported" }, ["TELEGRAM_BOT_TOKEN"]); + expect(state.legacy!.metadata.uid).toBe(uid); + expect(decode(state.legacy)).toEqual({ SLACK_BOT_TOKEN: "keep-slack", OPENAI_API_KEY: "legacy-supported" }); + }); + + it("does not treat a wrong workspace or same-name conflict as the legacy direct path", async () => { + const { state, execute, run } = cluster(); + state.namespace!.metadata.annotations[CLAIM.namespace] = "other-workspace"; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow(); + state.sandbox = undefined; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow("--namespace"); + expect(run.mock.calls.every(([, args]) => args[0] === "get")).toBe(true); + }); + + it("updates/removes owned source keys without delete/recreate or legacy fallback", async () => { + const { state, execute, run } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + const legacy = structuredClone(state.legacy); + await updateCredentialSource(execute, "demo", "kars-system", opts({ + updates: { SLACK_BOT_TOKEN: "new-slack" }, remove: ["TELEGRAM_BOT_TOKEN"], + })); + expect(decode(state.source)).toEqual({ SLACK_BOT_TOKEN: "new-slack" }); + expect(state.source!.metadata.uid).toBe(reference.uid); + expect(state.source!.metadata.annotations!["customer.example/note"]).toBe("preserve"); + expect(state.legacy).toEqual(legacy); + expect(run.mock.calls.some(([, args]) => args.includes("kars-demo") || args[0] === "delete")).toBe(false); + }); + + it("requires explicit --use-source when the pinned source incarnation was replaced", async () => { + const { state, execute } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.source.metadata.uid = "new-source"; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow("missing/replaced"); + await updateCredentialSource(execute, "demo", "kars-system", opts({ useSource: true })); + expect(state.sandbox!.spec.credentialsRef?.uid).toBe("new-source"); + }); + + it("disables the reference without deleting source/customer data and preserves no-restart on direct mode", async () => { + const { state, execute } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + const original = structuredClone(state.source); + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ restart: false }))).rejects.toThrow("--no-restart"); + await updateCredentialSource(execute, "demo", "kars-system", opts({ disableSource: true })); + expect(state.sandbox!.spec.credentialsRef).toBeUndefined(); + expect(state.source).toEqual(original); + expect(await updateCredentialSource(execute, "demo", "kars-system", opts({ restart: false }))).toBeUndefined(); + }); + + it("supports explicit pre-CR staging without automatic delivery", async () => { + const { state, execute } = cluster(); + state.sandbox = undefined; state.namespace = undefined; + const result = await updateCredentialSource(execute, "demo", "kars-system", opts({ + useSource: true, updates: { TELEGRAM_BOT_TOKEN: "staged" }, + })); + expect(result?.staged).toBe(true); + expect(state.sandbox).toBeUndefined(); + expect(state.namespace).toBeUndefined(); + }); +}); + +describe("credential authority and truthful errors", () => { + it.each(["purpose", "owner", "namespace"])("rejects conflicting %s without takeover", async variant => { + const { state, execute, run } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + if (variant === "purpose") state.source.metadata.annotations![SOURCE.purpose] = "other"; + if (variant === "owner") state.source.metadata.annotations![SOURCE.sandboxUid] = "old-sandbox"; + if (variant === "namespace") state.source.metadata.namespace = "other-workspace"; + const original = structuredClone(state.source); + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ useSource: true }))).rejects.toThrow(); + expect(state.source).toEqual(original); + expect(run.mock.calls.every(([, args]) => args[0] === "get")).toBe(true); + }); + + it("propagates non404 errors and rejects UID/RV races without exposing request bodies", async () => { + const { state, execute } = cluster(); + state.failRead = true; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow("Read credential resource failed"); + state.failRead = false; state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.failWrite = true; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ + updates: { TELEGRAM_BOT_TOKEN: "private-token-value" }, + }))).rejects.not.toThrow("private-token-value"); + state.failWrite = false; state.recreateSourceOnPatch = true; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ + updates: { TELEGRAM_BOT_TOKEN: "new-data" }, + }))).rejects.toThrow("Update credential source failed"); + expect(decode(state.source).TELEGRAM_BOT_TOKEN).toBe("initial-source"); + }); + + it("does not report activation when the controller has not acknowledged the source version", async () => { + const { state, execute } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.acknowledge = false; + await expect(waitForCredentialSource(execute, "demo", "kars-system", reference, { attempts: 1, delayMs: 0 })) + .rejects.toThrow("did not confirm"); + state.sandbox!.spec.credentialsRef = null; + await expect(verifyCredentialReference(execute, "demo", "kars-system", reference)).rejects.toThrow("did not retain"); + }); + + it("allows a repaired source to replace an older failure condition", async () => { + const { state, execute, run } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.acknowledge = false; + state.sandbox!.status = { conditions: [{ + type: "Degraded", status: "True", reason: "CredentialSourceUnavailable", message: "old failure", + }] }; + const original = run.getMockImplementation()!; + run.mockImplementation(async (...args) => { + const result = await original(...args); + if (args[1].includes("jsonpath={.metadata}")) state.acknowledge = true; + return result; + }); + await expect(waitForCredentialSource(execute, "demo", "kars-system", reference, { attempts: 2, delayMs: 0 })) + .resolves.toBeUndefined(); + }); +}); diff --git a/cli/src/lib/credential-source.ts b/cli/src/lib/credential-source.ts new file mode 100644 index 00000000..f7d3e3b8 --- /dev/null +++ b/cli/src/lib/credential-source.ts @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Execute } from "./deployment-target.js"; +import { CLAIM, namespaceClaimed, type OwnershipObject } from "./namespace-ownership.js"; +import { + SOURCE, decode, get, identity, parse, run, targetName, validateSource, validateValues, + type Metadata, type Sandbox, type Secret, type SourceRef, +} from "./credential-source-io.js"; + +export { FLAG_ENV, SOURCE_KEYS, removedKeys, updatesFromFlags } from "./credential-source-io.js"; +export type { SourceRef } from "./credential-source-io.js"; + +export interface PreparedSource { + reference: SourceRef; + sandbox?: Sandbox; +} + +function mergeValues(existing: Record, updates: Record, remove: string[]): Record { + const merged = { ...existing, ...updates }; + for (const key of remove) delete merged[key]; + return merged; +} + +async function legacyValues(execute: Execute, sandbox: Sandbox): Promise> { + const name = sandbox.metadata.name; + const target = `kars-${name}`; + const namespace = await get(execute, "namespace", target); + if (!namespace) { + if (sandbox.metadata.annotations?.["kars.azure.com/namespace-uid"]) { + throw new Error("Bound runtime namespace is missing; refusing an incomplete credential migration"); + } + return {}; + } + if (!namespaceClaimed(namespace, sandbox as unknown as OwnershipObject)) { + throw new Error("Credential migration requires namespace claim v1; finish namespace upgrade/adoption first"); + } + return decode(await get(execute, "secret", `${name}-credentials`, target)); +} + +async function writeSource( + execute: Execute, name: string, workspace: string, values: Record, existing?: Secret, sandbox?: Sandbox, +): Promise { + validateValues(values); + const sourceName = targetName(name, workspace); + const data: Record = Object.create(null); + for (const [key, value] of Object.entries(values)) data[key] = Buffer.from(value).toString("base64"); + let text: string; + if (existing) { + for (const key of Object.keys(existing.data ?? {})) if (!(key in values)) data[key] = null; + const { uid, resourceVersion } = existing.metadata; + text = await run(execute, "Update credential source", ["patch", "secret", sourceName, + "-n", workspace, "--type=merge", "--patch-file=/dev/stdin", "-o", "json"], { + metadata: { uid, resourceVersion }, data, + }); + } else { + text = await run(execute, "Create credential source", ["create", "-f", "-", "-o", "json"], { + apiVersion: "v1", kind: "Secret", type: "Opaque", + metadata: { + name: sourceName, namespace: workspace, + annotations: { + [SOURCE.purpose]: "agent-source-v1", [SOURCE.target]: name, + [SOURCE.workspace]: workspace, [SOURCE.intent]: "explicit-reference-v1", + }, + }, data, + }); + } + const result = parse(text); + identity(result.metadata); + if (result.metadata.name !== sourceName || result.metadata.namespace !== workspace + || (existing && result.metadata.uid !== existing.metadata.uid)) { + throw new Error("Credential source identity changed during write"); + } + validateSource(result, name, workspace, sandbox); + validateValues(decode(result)); + return { name: sourceName, uid: result.metadata.uid }; +} + +export async function prepareCredentialSource( + execute: Execute, name: string, workspace: string, updates: Record, + remove: string[] = [], useCurrentIncarnation = true, +): Promise { + const sourceName = targetName(name, workspace); + const sandbox = await get(execute, "karssandbox", name, workspace); + const reference = sandbox?.spec.credentialsRef; + if (reference && (reference.name !== sourceName || !reference.uid)) { + throw new Error("Sandbox credential reference is invalid; explicitly disable it before repair"); + } + const source = await get(execute, "secret", sourceName, workspace); + if (source) validateSource(source, name, workspace, sandbox); + if (reference && !useCurrentIncarnation && source?.metadata.uid !== reference.uid) { + throw new Error("Pinned credential source is missing/replaced; --use-source explicitly selects a new compatible incarnation"); + } + // Source mode is an entire collection. Migration is one-time and read-only + // against the existing target; active sources never fall back to legacy data. + const legacy = sandbox && !reference ? await legacyValues(execute, sandbox) : {}; + const values = mergeValues({ ...legacy, ...decode(source) }, updates, remove); + const selected = await writeSource(execute, name, workspace, values, source, sandbox); + return { reference: selected, sandbox }; +} + +export async function verifyCredentialReference( + execute: Execute, name: string, workspace: string, reference: SourceRef, +): Promise { + const sandbox = await get(execute, "karssandbox", name, workspace); + if (!sandbox || sandbox.spec.credentialsRef?.name !== reference.name + || sandbox.spec.credentialsRef.uid !== reference.uid) { + throw new Error("Cluster did not retain the pinned credential reference; upgrade the CRD/controller before using source mode"); + } + return sandbox; +} + +export async function applySourceSandbox( + execute: Execute, manifest: Record, prepared: PreparedSource, +): Promise { + const metadata = manifest.metadata as { name: string; namespace: string }; + const spec = { ...(manifest.spec as object), credentialsRef: prepared.reference }; + let written: Sandbox; + if (prepared.sandbox) { + const { uid, resourceVersion } = prepared.sandbox.metadata; + written = parse(await run(execute, "Bind existing Sandbox credentials", [ + "patch", "karssandbox", metadata.name, "-n", metadata.namespace, + "--type=merge", "--patch-file=/dev/stdin", "-o", "json", + ], { metadata: { uid, resourceVersion }, spec })); + if (written.metadata.uid !== uid) throw new Error("Sandbox identity changed while binding credentials"); + } else { + // CREATE, not apply: a racing Sandbox of the same name is not ours to bind. + written = parse(await run(execute, "Create source-bound Sandbox", ["create", "-f", "-", "-o", "json"], { + ...manifest, spec, + })); + } + identity(written.metadata); + const confirmed = await verifyCredentialReference(execute, metadata.name, metadata.namespace, prepared.reference); + if (confirmed.metadata.uid !== written.metadata.uid) throw new Error("Sandbox was recreated after source binding"); +} + +export async function waitForCredentialSource( + execute: Execute, name: string, workspace: string, reference: SourceRef, + options: { attempts?: number; delayMs?: number } = {}, +): Promise { + const attempts = options.attempts ?? 30; + for (let attempt = 0; attempt < attempts; attempt++) { + const sandbox = await verifyCredentialReference(execute, name, workspace, reference); + const text = await run(execute, "Read source reconciliation version", [ + "get", "secret", reference.name, "-n", workspace, "-o", "jsonpath={.metadata}", + ]); + const metadata = parse(text); + identity(metadata); + if (metadata.uid !== reference.uid) throw new Error("Credential source was replaced while awaiting reconciliation"); + const conditions = sandbox.status?.conditions ?? []; + const ready = conditions.find(condition => condition.type === "CredentialsReady" && condition.status === "True"); + if (ready) { + const observed = parse<{ sourceUid?: string; sourceVersion?: string }>(ready.message); + if (observed.sourceUid === reference.uid && observed.sourceVersion === metadata.resourceVersion) return; + } + // An earlier failure condition can precede this source revision. Allow the + // controller to process a repair rather than treating stale status as final. + if (attempt + 1 < attempts) await new Promise(resolve => setTimeout(resolve, options.delayMs ?? 3000)); + } + throw new Error("Controller did not confirm credential source reconciliation; check compatible controller/CRD versions and Sandbox conditions"); +} + +export async function updateCredentialSource( + execute: Execute, name: string, workspace: string, + options: { updates: Record; remove: string[]; useSource?: boolean; disableSource?: boolean; restart?: boolean }, +): Promise<{ kind: "source"; reference?: SourceRef; staged?: boolean } | undefined> { + targetName(name, workspace); + const sandbox = await get(execute, "karssandbox", name, workspace); + if (!sandbox?.spec.credentialsRef && !options.useSource && !options.disableSource) { + const namespace = await get(execute, "namespace", `kars-${name}`); + if (namespace?.metadata.annotations?.[CLAIM.version] !== undefined) { + if (!sandbox || !namespaceClaimed(namespace, sandbox as unknown as OwnershipObject)) { + throw new Error("Runtime namespace belongs to another or missing Sandbox; select its workspace with --namespace"); + } + } + return undefined; + } + if (options.restart === false) throw new Error("Source mode always refreshes runtime credentials; --no-restart is only supported for direct credentials"); + if (options.disableSource) { + if (!sandbox) throw new Error("Sandbox does not exist"); + if (options.useSource || options.remove.length || Object.keys(options.updates).length) { + throw new Error("--disable-source cannot be combined with credential changes"); + } + if (sandbox.spec.credentialsRef) { + const { uid, resourceVersion } = sandbox.metadata; + const result = parse(await run(execute, "Disable credential source", [ + "patch", "karssandbox", name, "-n", workspace, "--type=merge", "--patch-file=/dev/stdin", "-o", "json", + ], { metadata: { uid, resourceVersion }, spec: { credentialsRef: null } })); + if (result.spec.credentialsRef) throw new Error("Controller API did not remove the source reference"); + } + return { kind: "source" }; + } + const prepared = await prepareCredentialSource(execute, name, workspace, options.updates, options.remove, options.useSource === true); + if (prepared.sandbox && (prepared.sandbox.spec.credentialsRef?.uid !== prepared.reference.uid + || prepared.sandbox.spec.credentialsRef?.name !== prepared.reference.name)) { + await applySourceSandbox(execute, { + apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", + metadata: { name, namespace: workspace }, spec: prepared.sandbox.spec, + }, prepared); + } + if (prepared.sandbox) await waitForCredentialSource(execute, name, workspace, prepared.reference); + return { kind: "source", reference: prepared.reference, staged: !prepared.sandbox }; +} + +/** Preserve unrelated direct keys; explicit nulls remove keys across managers + * without deleting/recreating customer Secrets. */ +export async function updateDirectCredentials( + execute: Execute, name: string, updates: Record, remove: string[], +): Promise { + targetName(name, "kars-system"); + const namespace = `kars-${name}`; + const secretName = `${name}-credentials`; + const existing = await get(execute, "secret", secretName, namespace); + const data: Record = Object.create(null); + for (const [key, value] of Object.entries(updates)) data[key] = Buffer.from(value).toString("base64"); + for (const key of remove) data[key] = null; + if (existing) { + await run(execute, "Update direct credentials", [ + "patch", "secret", secretName, "-n", namespace, "--type=merge", "--patch-file=/dev/stdin", + ], { metadata: { uid: existing.metadata.uid, resourceVersion: existing.metadata.resourceVersion }, data }); + } else if (Object.keys(updates).some(key => !remove.includes(key))) { + for (const key of remove) delete data[key]; + await run(execute, "Create direct credentials", ["create", "-f", "-"], { + apiVersion: "v1", kind: "Secret", type: "Opaque", metadata: { name: secretName, namespace }, data, + }); + } +} diff --git a/cli/src/testing/credential-source.test.ts b/cli/src/testing/credential-source.test.ts new file mode 100644 index 00000000..015aedfd --- /dev/null +++ b/cli/src/testing/credential-source.test.ts @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { parseAllDocuments } from "yaml"; + +const root = new URL("../../../", import.meta.url); +const source = (path: string) => readFileSync(new URL(path, root), "utf8"); + +describe("credential source public integration", () => { + it("renders an optional UID-pinned schema with same-target and managed-runtime admission guards", () => { + const yaml = execFileSync("helm", [ + "template", "kars", fileURLToPath(new URL("deploy/helm/kars", root)), + "--show-only", "templates/crd.yaml", + ], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 }); + const resource = parseAllDocuments(yaml).map(doc => { + if (doc.errors.length) throw doc.errors[0]; + return doc.toJSON(); + }).find(doc => doc?.metadata?.name === "karssandboxes.kars.azure.com"); + const schema = resource.spec.versions[0].schema.openAPIV3Schema; + const spec = schema.properties.spec; + expect(spec.required).not.toContain("credentialsRef"); + expect(spec.properties.credentialsRef.required).toEqual(["name", "uid"]); + expect(schema["x-kubernetes-validations"].some((rule: { rule: string }) => + rule.rule.includes("self.metadata.name") && rule.rule.includes("kars-credential-source-"))).toBe(true); + expect(spec["x-kubernetes-validations"].some((rule: { rule: string }) => + rule.rule.includes("credentialsRef") && rule.rule.includes("overlay"))).toBe(true); + }); + + it("mounts the projection in the common agent container, never the inference router", () => { + const code = source("controller/src/reconciler/mod.rs"); + const agent = code.slice(code.indexOf("let mut agent_container ="), code.indexOf("let mut agent_container =") + 1200); + expect(agent).toContain('"envFrom": credentials.env_from(&name)'); + expect(code.match(/credentials\.env_from/g)).toHaveLength(1); + expect(code).toContain('"envFrom": inference::provider_env_from()'); + expect(code.indexOf("credential_sources::reconcile(")).toBeLessThan(code.indexOf("let runtime_spec =")); + expect(code).toContain(".owns("); + expect(code.split("\n").length).toBeLessThan(3530); + }); + + it("keeps source values off argv and binds through a real UID rather than a generated placeholder", () => { + const helpers = source("cli/src/lib/credential-source.ts"); + expect(helpers).toContain("metadata: { uid, resourceVersion }"); + expect(helpers).not.toContain("--from-literal"); + expect(helpers).not.toContain("--force"); + const add = source("cli/src/commands/add.ts"); + expect(add.indexOf("await prepareCredentialSource")).toBeLessThan(add.indexOf("await applySourceSandbox")); + expect(add).toContain("await waitForCredentialSource"); + const projection = source("controller/src/reconciler/credential_source_projection.rs"); + expect(projection).not.toContain(".force()"); + expect(projection).toContain("Preconditions"); + }); + + it("keeps new production modules bounded and free of stubs, crypto primitives, or gate waivers", () => { + for (const path of [ + "controller/src/credential_source.rs", + "controller/src/reconciler/agent_env.rs", + "controller/src/reconciler/credential_sources.rs", + "controller/src/reconciler/credential_source_projection.rs", + "controller/src/reconciler/credential_source_workloads.rs", + "cli/src/lib/credential-source.ts", "cli/src/lib/credential-source-io.ts", + ]) { + const text = source(path); + expect(text.split("\n").length - 1, path).toBeLessThanOrEqual(800); + expect(text.slice(0, 150), path).toContain("Copyright (c) Microsoft Corporation"); + expect(text, path).not.toMatch(/TODO\b|FIXME\b|XXX\b|HACK\b|unimplemented!\(|\btodo!\(|\bplaceholder\b|ci:stub-ok|ci:loc-ok/); + expect(text, path).not.toMatch(/^use (sha2|hmac|curve25519_dalek|ed25519_dalek|x25519_dalek|aes|chacha20poly1305)::/m); + expect(text, path).not.toMatch(/crypto\.subtle\.sign|createHmac|createSign|@noble\/(curves|hashes)/); + } + }); +}); diff --git a/controller/src/crd.rs b/controller/src/crd.rs index 8eb458c1..36c89d02 100644 --- a/controller/src/crd.rs +++ b/controller/src/crd.rs @@ -72,6 +72,11 @@ pub struct KarsSandboxSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_ref: Option, + /// Explicit agent credential source in this CR's workspace. Source mode + /// replaces the legacy credential collection; no cross-namespace refs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credentials_ref: Option, + /// Network policy pub network_policy: Option, diff --git a/controller/src/credential_source.rs b/controller/src/credential_source.rs new file mode 100644 index 00000000..d6267529 --- /dev/null +++ b/controller/src/credential_source.rs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Public, opt-in agent credential-source contract. This is not a generic +//! Secret reference: the reserved name, purpose, target, and UID are mandatory. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct CredentialSourceRef { + #[schemars( + length(min = 1, max = 253), + regex(pattern = "^kars-credential-source-[a-z0-9][a-z0-9-]*$") + )] + pub name: String, + #[schemars(length(min = 1, max = 128), regex(pattern = "^[A-Za-z0-9-]+$"))] + pub uid: String, +} + +pub const SOURCE_PREFIX: &str = "kars-credential-source-"; +pub const PURPOSE: &str = "kars.azure.com/credential-purpose"; +pub const TARGET: &str = "kars.azure.com/credential-target"; +pub const INTENT: &str = "kars.azure.com/credential-binding-intent"; +pub const SANDBOX_UID: &str = "kars.azure.com/credential-sandbox-uid"; +pub const WORKSPACE: &str = "kars.azure.com/credential-workspace"; +pub const NAMESPACE_UID: &str = "kars.azure.com/credential-namespace-uid"; +pub const SOURCE_UID: &str = "kars.azure.com/credential-source-uid"; +pub const PROJECTION_UID: &str = "kars.azure.com/credential-projection-uid"; +pub const SOURCE_PURPOSE: &str = "agent-source-v1"; +pub const PROJECTION_PURPOSE: &str = "agent-projection-v1"; +pub const BINDING_INTENT: &str = "explicit-reference-v1"; +pub const POD_VERSION: &str = "kars.azure.com/credential-projection-version"; + +// Same agent-only allowlist as the existing handoff credential transport. +// OPENAI_API_KEY and other inference/control-plane variables are deliberately +// absent; those remain router/provider configuration, not agent credentials. +pub const AGENT_KEYS: &[&str] = &[ + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_ALLOW_FROM", + "SLACK_BOT_TOKEN", + "DISCORD_BOT_TOKEN", + "WHATSAPP_ENABLED", + "BRAVE_API_KEY", + "TAVILY_API_KEY", + "EXA_API_KEY", + "FIRECRAWL_API_KEY", + "PERPLEXITY_API_KEY", +]; + +pub fn source_name(sandbox: &str) -> String { + format!("{SOURCE_PREFIX}{sandbox}") +} + +pub fn projection_name(sandbox: &str) -> String { + format!("{sandbox}-credential-projection") +} + +pub fn valid_ref(reference: &CredentialSourceRef, sandbox: &str) -> bool { + reference.name == source_name(sandbox) + && !reference.uid.is_empty() + && reference.uid.len() <= 128 + && reference + .uid + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::crd::KarsSandbox; + use kube::CustomResourceExt; + + #[test] + fn reference_identity_is_retained_in_serialized_spec_digests() { + let mut spec = crate::crd::KarsSandboxSpec { + credentials_ref: Some(CredentialSourceRef { + name: source_name("demo"), + uid: "source-a".into(), + }), + ..Default::default() + }; + let first = crate::providers::signing::sha256_hex(&serde_json::to_vec(&spec).unwrap()); + spec.credentials_ref.as_mut().unwrap().uid = "source-b".into(); + let second = crate::providers::signing::sha256_hex(&serde_json::to_vec(&spec).unwrap()); + assert_ne!(first, second); + } + + #[test] + fn reserved_reference_is_uid_pinned_and_not_an_arbitrary_secret_path() { + let mut reference = CredentialSourceRef { + name: source_name("demo"), + uid: "source-uid".into(), + }; + assert!(valid_ref(&reference, "demo")); + assert!(!valid_ref(&reference, "other")); + reference.name = "controller-receipt-identity".into(); + assert!(!valid_ref(&reference, "demo")); + reference.name = source_name("demo"); + reference.uid = "../source".into(); + assert!(!valid_ref(&reference, "demo")); + } + + #[test] + fn generated_schema_keeps_legacy_specs_unchanged_and_requires_both_reference_fields() { + let spec = crate::crd::KarsSandboxSpec::default(); + assert!( + serde_json::to_value(spec) + .unwrap() + .get("credentialsRef") + .is_none() + ); + let crd = serde_json::to_value(KarsSandbox::crd()).unwrap(); + let schema = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let required = schema["properties"]["credentialsRef"]["required"] + .as_array() + .unwrap(); + assert!(required.contains(&serde_json::json!("name"))); + assert!(required.contains(&serde_json::json!("uid"))); + assert!( + !schema["required"] + .as_array() + .unwrap() + .contains(&serde_json::json!("credentialsRef")) + ); + } +} diff --git a/controller/src/main.rs b/controller/src/main.rs index 0ec22660..6a8a9bb4 100644 --- a/controller/src/main.rs +++ b/controller/src/main.rs @@ -27,6 +27,7 @@ mod crd; #[allow(dead_code)] // CRD-installation pipeline (Phase 1 close-out + future kubectl-claw-attest) consumes these helpers. mod crd_validations; +mod credential_source; mod egress_allowlist_compile; mod egress_approval; mod egress_approval_compile; diff --git a/controller/src/reconciler/agent_env.rs b/controller/src/reconciler/agent_env.rs new file mode 100644 index 00000000..b1975df0 --- /dev/null +++ b/controller/src/reconciler/agent_env.rs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::runtime::RuntimeDeploymentPlan; +use serde_json::{Value, json}; + +pub const RESERVED_PREFIXES: &[&str] = &["AGT_", "FOUNDRY_AGENT_", "AZURE_", "IMDS_", "KARS_"]; + +pub fn merge(env: &mut Vec, plan: &RuntimeDeploymentPlan) { + let mut existing: std::collections::HashSet = env + .iter() + .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) + .collect(); + for (key, value) in &plan.runtime_extra_env { + if key.is_empty() + || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + || key.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + tracing::warn!(key = %key, "extraEnv: invalid env var name, skipping"); + continue; + } + if RESERVED_PREFIXES + .iter() + .any(|prefix| key.starts_with(prefix)) + { + tracing::warn!(key = %key, "extraEnv: key uses reserved prefix, skipping"); + continue; + } + if value.contains('\0') { + tracing::warn!(key = %key, "extraEnv: value contains NUL byte, skipping"); + continue; + } + if !existing.insert(key.clone()) { + tracing::debug!(key = %key, "extraEnv: overridden by reconciler, skipping"); + continue; + } + env.push(json!({"name": key, "value": value})); + } + for entry in &plan.raw_env { + let Some(name) = entry.get("name").and_then(|n| n.as_str()) else { + tracing::warn!("rawEnv: entry missing `name`, skipping"); + continue; + }; + if name.is_empty() + || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + || name.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + tracing::warn!(key = %name, "rawEnv: invalid env var name, skipping"); + continue; + } + if RESERVED_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + { + tracing::warn!(key = %name, "rawEnv: key uses reserved prefix, skipping"); + continue; + } + if !existing.insert(name.into()) { + tracing::debug!(key = %name, "rawEnv: overridden by reconciler, skipping"); + continue; + } + env.push(entry.clone()); + } +} diff --git a/controller/src/reconciler/credential_source_projection.rs b/controller/src/reconciler/credential_source_projection.rs new file mode 100644 index 00000000..fed89bf2 --- /dev/null +++ b/controller/src/reconciler/credential_source_projection.rs @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use kube::api::{DeleteParams, Preconditions}; + +fn owner(ns: &Namespace) -> Result { + Ok(OwnerReference { + api_version: "v1".into(), + kind: "Namespace".into(), + name: ns.name_any(), + uid: identity(&ns.metadata)?.0.into(), + controller: Some(true), + block_owner_deletion: Some(false), + }) +} + +fn validate_owner(meta: &ObjectMeta, sandbox: &KarsSandbox, ns: &Namespace) -> Result<(), Error> { + identity(meta)?; + if meta.name.as_deref() != Some(projection_name(&sandbox.name_any()).as_str()) + || meta.namespace.as_deref() != Some(ns.name_any().as_str()) + || meta.deletion_timestamp.is_some() + || annotation(meta, PURPOSE) != Some(PROJECTION_PURPOSE) + || annotation(meta, TARGET) != sandbox.metadata.name.as_deref() + || annotation(meta, WORKSPACE) != sandbox.metadata.namespace.as_deref() + || annotation(meta, SANDBOX_UID) != sandbox.metadata.uid.as_deref() + || annotation(meta, NAMESPACE_UID) != ns.metadata.uid.as_deref() + || meta.owner_references.as_deref() != Some([owner(ns)?].as_slice()) + { + return Err(Error::Invalid( + "credential projection is not owned by this Sandbox/namespace", + )); + } + Ok(()) +} + +pub(super) fn validate( + meta: &ObjectMeta, + sandbox: &KarsSandbox, + ns: &Namespace, +) -> Result<(), Error> { + validate_owner(meta, sandbox, ns)?; + if annotation(meta, PROJECTION_UID) != Some(identity(meta)?.0) { + return Err(Error::Invalid( + "projection incarnation is not sealed to its UID", + )); + } + Ok(()) +} + +async fn seal( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + meta: &ObjectMeta, +) -> Result, Error> { + namespace_current(client, sandbox, ns).await?; + sandbox_current(client, sandbox).await?; + let (uid, rv) = identity(meta)?; + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let sealed = api.patch_metadata(&projection_name(&sandbox.name_any()), &PatchParams::default(), &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv, "annotations": {PROJECTION_UID: uid}} + }))).await.map_err(|e| api_error("seal projection anchor UID", e))?; + validate(&sealed.metadata, sandbox, ns)?; + Ok(sealed) +} + +pub(super) async fn anchor( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + source: &Secret, +) -> Result, Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let name = projection_name(&sandbox.name_any()); + if let Some(existing) = metadata(&api, &name).await? { + validate_owner(&existing.metadata, sandbox, ns)?; + if annotation(&existing.metadata, PROJECTION_UID).is_some() { + validate(&existing.metadata, sandbox, ns)?; + return Ok(existing); + } + let anchor = api + .get(&name) + .await + .map_err(|e| api_error("inspect unfinished projection anchor", e))?; + let authored = anchor + .metadata + .managed_fields + .as_ref() + .is_some_and(|fields| { + fields.iter().any(|field| { + field.manager.as_deref() == Some(crate::field_managers::CLAWSANDBOX) + }) + }); + if identity(&anchor.metadata)? != identity(&existing.metadata)? + || anchor.type_.as_deref() != Some("Opaque") + || anchor.data.as_ref().is_some_and(|data| !data.is_empty()) + || !authored + { + return Err(Error::Invalid( + "unsealed projection is not an empty controller anchor", + )); + } + return seal(client, sandbox, ns, &anchor.metadata).await; + } + namespace_current(client, sandbox, ns).await?; + sandbox_current(client, sandbox).await?; + let anchor: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": { + "name": name, "namespace": ns.name_any(), + "annotations": { + PURPOSE: PROJECTION_PURPOSE, TARGET: sandbox.name_any(), + WORKSPACE: sandbox.namespace(), SANDBOX_UID: sandbox.metadata.uid, + NAMESPACE_UID: ns.metadata.uid, SOURCE_UID: source.metadata.uid + }, + "ownerReferences": [owner(ns)?] + } + })) + .map_err(|_| Error::Invalid("projection metadata serialization failed"))?; + let created = api + .create( + &PostParams { + field_manager: Some(crate::field_managers::CLAWSANDBOX.into()), + ..Default::default() + }, + &anchor, + ) + .await + .map_err(|e| api_error("create projection anchor", e))?; + validate_owner(&created.metadata, sandbox, ns)?; + seal(client, sandbox, ns, &created.metadata).await +} + +pub(super) async fn revoke( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + remove: bool, +) -> Result<(), Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let name = projection_name(&sandbox.name_any()); + let Some(meta) = metadata(&api, &name).await? else { + return Ok(()); + }; + if validate(&meta.metadata, sandbox, ns).is_err() { + // Never clear/delete an unrelated object, even when its reserved name + // collides with a previously used projection. + return Ok(()); + } + namespace_current(client, sandbox, ns).await?; + let existing = api + .get(&name) + .await + .map_err(|e| api_error("read owned projection for revocation", e))?; + if identity(&existing.metadata)? != identity(&meta.metadata)? + || existing.type_.as_deref() != Some("Opaque") + { + return Err(Error::Invalid( + "projection identity/type changed before revocation", + )); + } + let (uid, rv) = identity(&existing.metadata)?; + if remove { + match api + .delete( + &name, + &DeleteParams { + preconditions: Some(Preconditions { + uid: Some(uid.into()), + resource_version: Some(rv.into()), + }), + ..Default::default() + }, + ) + .await + { + Ok(_) => {} + Err(kube::Error::Api(status)) if status.code == 404 => {} + Err(error) => return Err(api_error("remove owned projection", error)), + } + } else if existing.data.as_ref().is_some_and(|data| !data.is_empty()) { + // JSON merge null clears all keys; {} alone would preserve old keys. + api.patch_metadata( + &name, + &PatchParams::default(), + &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv}, "data": null + })), + ) + .await + .map_err(|e| api_error("revoke owned projection", e))?; + } + Ok(()) +} + +pub(super) async fn detach( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, +) -> Result<(), Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let owned = metadata(&api, &projection_name(&sandbox.name_any())) + .await? + .is_some_and(|value| validate(&value.metadata, sandbox, ns).is_ok()); + workloads::pause(client, sandbox, ns, !owned).await?; + if owned { + revoke(client, sandbox, ns, true).await?; + } + Ok(()) +} diff --git a/controller/src/reconciler/credential_source_test_server.rs b/controller/src/reconciler/credential_source_test_server.rs new file mode 100644 index 00000000..9c250668 --- /dev/null +++ b/controller/src/reconciler/credential_source_test_server.rs @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use crate::crd::KarsSandboxSpec; + +pub const SOURCE_PATH: &str = "/api/v1/namespaces/workspace-a/secrets/kars-credential-source-demo"; +pub const TARGETS_PATH: &str = "/api/v1/namespaces/kars-demo/secrets"; +pub const TARGET_PATH: &str = "/api/v1/namespaces/kars-demo/secrets/demo-credential-projection"; +pub const NS_PATH: &str = "/api/v1/namespaces/kars-demo"; +const SANDBOX_PATH: &str = + "/apis/kars.azure.com/v1alpha1/namespaces/workspace-a/karssandboxes/demo"; +const DEPLOYMENT_PATH: &str = "/apis/apps/v1/namespaces/kars-demo/deployments/demo"; + +pub fn sandbox() -> KarsSandbox { + let mut sandbox = KarsSandbox::new("demo", KarsSandboxSpec::default()); + sandbox.metadata.namespace = Some("workspace-a".into()); + sandbox.metadata.uid = Some("sandbox-a".into()); + sandbox.metadata.resource_version = Some("10".into()); + sandbox.metadata.generation = Some(1); + sandbox.spec.credentials_ref = Some(CredentialSourceRef { + name: source_name("demo"), + uid: "source-a".into(), + }); + sandbox.annotations_mut().insert( + super::super::super::namespace_ownership::NAMESPACE_UID.into(), + "namespace-a".into(), + ); + sandbox +} + +pub fn namespace() -> Namespace { + use crate::reconciler::namespace_ownership as ownership; + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Namespace", + "metadata": { + "name": "kars-demo", "uid": "namespace-a", "resourceVersion": "20", + "annotations": { + (ownership::VERSION): "v1", (ownership::SOURCE_NAME): "demo", + (ownership::SOURCE_NAMESPACE): "workspace-a", (ownership::SOURCE_UID): "sandbox-a" + } + } + })) + .unwrap() +} + +pub fn source() -> Secret { + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": { + "name": source_name("demo"), "namespace": "workspace-a", + "uid": "source-a", "resourceVersion": "30", + "annotations": { + PURPOSE: SOURCE_PURPOSE, TARGET: "demo", WORKSPACE: "workspace-a", INTENT: BINDING_INTENT + } + }, + "data": {"TELEGRAM_BOT_TOKEN": ByteString(b"initial".to_vec())} + })).unwrap() +} + +pub fn deployment() -> Deployment { + serde_json::from_value(json!({ + "apiVersion": "apps/v1", "kind": "Deployment", + "metadata": { + "name": "demo", "namespace": "kars-demo", "uid": "deployment-a", "resourceVersion": "40", + "labels": { + "kars.azure.com/sandbox": "demo", "kars.azure.com/component": "sandbox", + "kars.azure.com/parent-namespace": "workspace-a" + }, + "managedFields": [{ + "manager": crate::field_managers::CLAWSANDBOX, "operation": "Apply", "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", "fieldsV1": {"f:spec": {}} + }] + }, + "spec": { + "replicas": 1, "selector": {"matchLabels": {"kars.azure.com/sandbox": "demo"}}, + "template": {"metadata": {}, "spec": {"containers": [{ + "name": "openclaw", "image": "agent:latest", + "envFrom": [{"secretRef": {"name": "demo-credentials", "optional": true}}] + }]}} + } + })).unwrap() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Fault { + None, + ReplaceNamespaceAfterAnchor, + ReplaceProjectionOnWrite, + ChangeSourceAfterAnchor, + FailValueWrite, + CreateConflict, + SourceReadForbidden, + FailSealOnce, +} + +pub struct State { + pub sandbox: KarsSandbox, + pub namespace: Namespace, + pub source: Option, + pub projection: Option, + pub deployment: Option, + pub fault: Fault, + pub successful_value_writes: usize, + revision: usize, +} + +impl State { + pub fn new() -> Self { + Self { + sandbox: sandbox(), + namespace: namespace(), + source: Some(source()), + projection: None, + deployment: Some(deployment()), + fault: Fault::None, + successful_value_writes: 0, + revision: 100, + } + } +} + +fn response(code: u16, body: Value) -> ResponseTemplate { + ResponseTemplate::new(code).set_body_json(body) +} + +fn failure(code: u16) -> ResponseTemplate { + response( + code, + json!({ + "apiVersion": "v1", "kind": "Status", "status": "Failure", "reason": "Failure", "code": code, + "message": "admission body contains secret-never-log-this" + }), + ) +} + +fn output(request: &Request, value: Value) -> ResponseTemplate { + if request + .headers + .get("accept") + .is_some_and(|header| header.to_str().unwrap().contains("PartialObjectMetadata")) + { + response( + 200, + json!({"apiVersion": "meta.k8s.io/v1", "kind": "PartialObjectMetadata", "metadata": value["metadata"]}), + ) + } else { + response(200, value) + } +} + +fn merge(target: &mut Value, patch: Value) { + if let Value::Object(patch) = patch { + if !target.is_object() { + *target = json!({}); + } + for (key, value) in patch { + if value.is_null() { + target.as_object_mut().unwrap().remove(&key); + } else { + merge( + target + .as_object_mut() + .unwrap() + .entry(key) + .or_insert(Value::Null), + value, + ); + } + } + } else { + *target = patch; + } +} + +pub fn nonempty_value_write(request: &Request) -> bool { + if request.method != "PATCH" || request.url.path() != TARGET_PATH { + return false; + } + let body: Value = serde_json::from_slice(&request.body).unwrap(); + body.get("data") + .and_then(Value::as_object) + .is_some_and(|values| values.values().any(|value| !value.is_null())) +} + +#[derive(Clone)] +struct Server(Arc>); + +impl Respond for Server { + fn respond(&self, request: &Request) -> ResponseTemplate { + let mut state = self.0.lock().unwrap(); + let path = request.url.path(); + if request.method == "GET" { + let value = match path { + SANDBOX_PATH => Some(serde_json::to_value(&state.sandbox).unwrap()), + NS_PATH => Some(serde_json::to_value(&state.namespace).unwrap()), + SOURCE_PATH => { + if state.fault == Fault::SourceReadForbidden { + return failure(403); + } + state + .source + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()) + } + TARGET_PATH => state + .projection + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + DEPLOYMENT_PATH => state + .deployment + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + _ => None, + }; + return value.map_or_else(|| failure(404), |value| output(request, value)); + } + let body: Value = serde_json::from_slice(&request.body).unwrap(); + if request.method == "POST" && path == TARGETS_PATH { + if state.fault == Fault::CreateConflict || state.projection.is_some() { + return failure(409); + } + assert!(body.get("data").is_none() && body.get("stringData").is_none()); + let mut value = body; + value["metadata"]["uid"] = json!("projection-uid"); + value["metadata"]["resourceVersion"] = json!("100"); + value["metadata"]["managedFields"] = json!([{ + "manager": crate::field_managers::CLAWSANDBOX, "operation": "Update", + "apiVersion": "v1", "fieldsType": "FieldsV1", "fieldsV1": {"f:metadata": {}} + }]); + state.projection = Some(serde_json::from_value(value.clone()).unwrap()); + match state.fault { + Fault::ReplaceNamespaceAfterAnchor => { + state.namespace.metadata.uid = Some("replacement".into()) + } + Fault::ChangeSourceAfterAnchor => { + state.source.as_mut().unwrap().metadata.resource_version = + Some("changed".into()) + } + _ => {} + } + return output(request, value); + } + if request.method == "DELETE" && path == TARGET_PATH { + let Some(existing) = state.projection.as_ref() else { + return failure(404); + }; + if body["preconditions"]["uid"] != json!(existing.metadata.uid) + || body["preconditions"]["resourceVersion"] + != json!(existing.metadata.resource_version) + { + return failure(409); + } + return response( + 200, + serde_json::to_value(state.projection.take().unwrap()).unwrap(), + ); + } + if request.method == "PATCH" { + let value_write = nonempty_value_write(request); + if path == TARGET_PATH + && body["metadata"]["annotations"] + .get(PROJECTION_UID) + .is_some() + && state.fault == Fault::FailSealOnce + { + state.fault = Fault::None; + return failure(500); + } + if value_write && state.fault == Fault::ReplaceProjectionOnWrite { + state.projection.as_mut().unwrap().metadata.uid = Some("foreign-projection".into()); + state.projection.as_mut().unwrap().metadata.annotations = None; + } + if value_write && state.fault == Fault::FailValueWrite { + return failure(409); + } + let mut existing = match path { + SOURCE_PATH => state + .source + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + TARGET_PATH => state + .projection + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + DEPLOYMENT_PATH => state + .deployment + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + _ if path == format!("{SANDBOX_PATH}/status") => { + Some(serde_json::to_value(&state.sandbox).unwrap()) + } + _ => None, + }; + let Some(ref mut value) = existing else { + return failure(404); + }; + if body["metadata"]["uid"] != value["metadata"]["uid"] + || body["metadata"]["resourceVersion"] != value["metadata"]["resourceVersion"] + { + return failure(409); + } + merge(value, body); + state.revision += 1; + value["metadata"]["resourceVersion"] = json!(state.revision.to_string()); + match path { + SOURCE_PATH => state.source = Some(serde_json::from_value(value.clone()).unwrap()), + TARGET_PATH => { + if value_write { + state.successful_value_writes += 1; + } + state.projection = Some(serde_json::from_value(value.clone()).unwrap()); + } + DEPLOYMENT_PATH => { + state.deployment = Some(serde_json::from_value(value.clone()).unwrap()) + } + _ => state.sandbox = serde_json::from_value(value.clone()).unwrap(), + } + return output(request, value.clone()); + } + failure(500) + } +} + +pub async fn setup(state: State) -> (MockServer, Client, Arc>) { + let server = MockServer::start().await; + let state = Arc::new(Mutex::new(state)); + Mock::given(wiremock::matchers::any()) + .respond_with(Server(state.clone())) + .mount(&server) + .await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let client = Client::try_from(kube::Config::new(server.uri().parse().unwrap())).unwrap(); + (server, client, state) +} + +pub fn resume(state: &Arc>, mode: &Mode) { + let mut state = state.lock().unwrap(); + let sandbox = state.sandbox.clone(); + let ns = state.namespace.clone(); + let deployment = state.deployment.as_mut().unwrap(); + deployment.spec.as_mut().unwrap().replicas = Some(1); + mode.decorate(deployment, &sandbox, &ns); +} diff --git a/controller/src/reconciler/credential_source_tests.rs b/controller/src/reconciler/credential_source_tests.rs new file mode 100644 index 00000000..f1fb1b5c --- /dev/null +++ b/controller/src/reconciler/credential_source_tests.rs @@ -0,0 +1,701 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use std::sync::{Arc, Mutex}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + +#[path = "credential_source_test_server.rs"] +mod server; +use server::*; + +#[test] +fn provider_control_plane_and_process_environment_keys_are_not_credential_sources() { + for key in [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "AZURE_CLIENT_SECRET", + "AGT_SIGNING_KEY", + "KARS_ADMIN_TOKEN", + "NODE_OPTIONS", + "PATH", + "token", + "CUSTOM_TOKEN", + ] { + let mut secret = source(); + secret.data = Some(BTreeMap::from([( + key.into(), + ByteString(b"sensitive".to_vec()), + )])); + assert!(validate_values(&secret).is_err(), "{key}"); + } + for bytes in [vec![0], vec![255]] { + let mut secret = source(); + secret.data = Some(BTreeMap::from([( + "TELEGRAM_BOT_TOKEN".into(), + ByteString(bytes), + )])); + assert!(validate_values(&secret).is_err()); + } +} + +#[test] +fn mode_preserves_legacy_shape_and_keeps_projection_values_out_of_pod_specs() { + assert_eq!( + Mode::Legacy.env_from("demo"), + json!([ + {"secretRef": {"name": "demo-credentials", "optional": true}} + ]) + ); + let mut deployment = deployment(); + let original = deployment.clone(); + Mode::Legacy.decorate(&mut deployment, &sandbox(), &namespace()); + assert_eq!(deployment, original); + let mode = Mode::Source { + uid: "projection-uid".into(), + version: "42".into(), + source_uid: "source-a".into(), + source_version: "30".into(), + }; + mode.decorate(&mut deployment, &sandbox(), &namespace()); + assert_eq!( + mode.env_from("demo"), + json!([ + {"secretRef": {"name": "demo-credential-projection", "optional": false}} + ]) + ); + assert_eq!( + deployment.spec.unwrap().strategy.unwrap().type_.as_deref(), + Some("Recreate") + ); +} + +#[test] +fn errors_never_echo_api_bodies_or_credential_values() { + let error = api_error( + "write", + kube::Error::Api(Box::new( + serde_json::from_value(json!({ + "status": "Failure", "reason": "Invalid", "code": 422, + "message": "stringData: secret-never-log-this" + })) + .unwrap(), + )), + ); + assert!(!format!("{error:?} {error}").contains("secret-never-log-this")); +} + +#[test] +fn metadata_watch_owner_is_the_explicitly_bound_sandbox_uid() { + let reference = source_owner(&sandbox()).unwrap(); + assert_eq!(reference.name, "demo"); + assert_eq!(reference.uid, "sandbox-a"); + assert_eq!(reference.controller, Some(true)); +} + +#[tokio::test] +async fn absent_reference_leaves_legacy_credentials_and_workloads_unchanged() { + let mut state = State::new(); + state.sandbox.spec.credentials_ref = None; + let original = state.deployment.clone(); + let sandbox = state.sandbox.clone(); + let (server, client, state) = setup(state).await; + assert!(matches!( + reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .unwrap(), + Mode::Legacy + )); + assert_eq!(state.lock().unwrap().deployment, original); + assert!( + server + .received_requests() + .await + .unwrap() + .iter() + .all(|r| r.method == "GET") + ); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(|r| r.url.path() == SOURCE_PATH) + ); +} + +#[tokio::test] +async fn first_source_creation_binds_uid_then_fences_values_behind_an_empty_anchor() { + let (server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert!(matches!(mode, Mode::Source { .. })); + let requests = server.received_requests().await.unwrap(); + let anchor = requests + .iter() + .position(|r| r.method == "POST" && r.url.path() == TARGETS_PATH) + .unwrap(); + let anchor_body: Value = serde_json::from_slice(&requests[anchor].body).unwrap(); + assert!(anchor_body.get("data").is_none() && anchor_body.get("stringData").is_none()); + let value_write = requests.iter().position(nonempty_value_write).unwrap(); + assert!(anchor < value_write); + assert!( + requests[anchor + 1..value_write] + .iter() + .any(|r| r.url.path() == NS_PATH) + ); + assert!( + requests[anchor + 1..value_write] + .iter() + .any(|r| r.url.path() == SOURCE_PATH) + ); + let body: Value = serde_json::from_slice(&requests[value_write].body).unwrap(); + assert_eq!(body["metadata"]["uid"], "projection-uid"); + assert!(body["metadata"]["resourceVersion"].is_string()); + let state = state.lock().unwrap(); + assert_eq!( + state.projection.as_ref().unwrap().data, + state.source.as_ref().unwrap().data + ); + assert_eq!( + state + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); + assert_eq!( + annotation(&state.source.as_ref().unwrap().metadata, SANDBOX_UID), + Some("sandbox-a") + ); +} + +#[tokio::test] +async fn source_key_updates_and_removal_refresh_without_resurrecting_legacy_values() { + let (_server, client, state) = setup(State::new()).await; + let first = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &first); + { + let mut state = state.lock().unwrap(); + let source = state.source.as_mut().unwrap(); + source.data = Some(BTreeMap::from([( + "SLACK_BOT_TOKEN".into(), + ByteString(b"rotated".to_vec()), + )])); + source.metadata.resource_version = Some("source-rotated".into()); + } + let next = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert_ne!(format!("{first:?}"), format!("{next:?}")); + let state = state.lock().unwrap(); + let data = state.projection.as_ref().unwrap().data.as_ref().unwrap(); + assert!(!data.contains_key("TELEGRAM_BOT_TOKEN")); + assert_eq!(data["SLACK_BOT_TOKEN"].0, b"rotated"); + assert_eq!( + state + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); +} + +#[tokio::test] +async fn metadata_only_source_changes_do_not_roll_pods_or_rewrite_projection_values() { + let (server, client, state) = setup(State::new()).await; + let first = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &first); + state + .lock() + .unwrap() + .source + .as_mut() + .unwrap() + .metadata + .resource_version = Some("metadata-change".into()); + let before = server.received_requests().await.unwrap().len(); + let next = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + let mut before_deployment = deployment(); + let mut after_deployment = deployment(); + first.decorate(&mut before_deployment, &sandbox(), &namespace()); + next.decorate(&mut after_deployment, &sandbox(), &namespace()); + assert_eq!(before_deployment, after_deployment); + assert!( + server.received_requests().await.unwrap()[before..] + .iter() + .all(|r| r.method == "GET") + ); +} + +#[tokio::test] +async fn missing_replaced_or_invalid_sources_stop_runtime_and_revoke_only_owned_projection() { + for variant in ["missing", "replaced", "purpose", "owner", "provider-key"] { + let (_server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &mode); + { + let mut state = state.lock().unwrap(); + match variant { + "missing" => state.source = None, + "replaced" => { + state.source.as_mut().unwrap().metadata.uid = Some("new-source".into()) + } + "purpose" => { + state + .source + .as_mut() + .unwrap() + .annotations_mut() + .insert(PURPOSE.into(), "other".into()); + } + "owner" => { + state + .source + .as_mut() + .unwrap() + .annotations_mut() + .insert(SANDBOX_UID.into(), "other".into()); + } + _ => { + state.source.as_mut().unwrap().data = Some(BTreeMap::from([( + "OPENAI_API_KEY".into(), + ByteString(b"provider-secret".to_vec()), + )])) + } + } + } + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err(), + "{variant}" + ); + let state = state.lock().unwrap(); + assert!( + state + .projection + .as_ref() + .unwrap() + .data + .as_ref() + .is_none_or(BTreeMap::is_empty), + "{variant}" + ); + assert_eq!( + state + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); + assert_eq!( + state.sandbox.status.as_ref().unwrap().phase.as_deref(), + Some("Degraded") + ); + } +} + +#[tokio::test] +async fn explicit_reference_removal_revokes_projection_and_preserves_legacy_secret() { + let (_server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &mode); + let current = { + let mut state = state.lock().unwrap(); + state.sandbox.spec.credentials_ref = None; + state.sandbox.clone() + }; + assert!(matches!( + reconcile(&client, ¤t, Some(&namespace()), "agent:latest") + .await + .unwrap(), + Mode::Legacy + )); + assert!(state.lock().unwrap().projection.is_none()); + assert_eq!( + state + .lock() + .unwrap() + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); +} + +#[tokio::test] +async fn foreign_projection_and_namespace_never_receive_values_or_get_taken_over() { + for variant in [ + "projection", + "namespace", + "namespace-replaced", + "source-cross-workspace", + ] { + let mut state = State::new(); + match variant { + "projection" => { + let mut foreign = source(); + foreign.metadata.name = Some(projection_name("demo")); + foreign.metadata.namespace = Some("kars-demo".into()); + foreign.metadata.uid = Some("foreign-projection".into()); + state.projection = Some(foreign); + } + "namespace" => { + state.namespace.annotations_mut().insert( + super::super::namespace_ownership::SOURCE_NAMESPACE.into(), + "other-workspace".into(), + ); + } + "namespace-replaced" => state.namespace.metadata.uid = Some("replacement".into()), + _ => state.source.as_mut().unwrap().metadata.namespace = Some("other-workspace".into()), + } + let original = state.projection.clone(); + let (server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(nonempty_value_write) + ); + if variant == "projection" { + assert_eq!(state.lock().unwrap().projection, original); + } + } +} + +#[tokio::test] +async fn namespace_source_and_destination_races_never_write_values_after_failed_fences() { + for fault in [ + Fault::ReplaceNamespaceAfterAnchor, + Fault::ReplaceProjectionOnWrite, + Fault::ChangeSourceAfterAnchor, + Fault::FailValueWrite, + Fault::CreateConflict, + Fault::SourceReadForbidden, + ] { + let mut state = State::new(); + state.fault = fault; + let (_server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert_eq!(state.lock().unwrap().successful_value_writes, 0); + } +} + +#[test] +fn chart_schema_and_generated_reference_contract_agree() { + use kube::CustomResourceExt; + use serde::Deserialize; + let document = serde_yaml::Deserializer::from_str(include_str!( + "../../../deploy/helm/kars/templates/crd.yaml" + )) + .next() + .unwrap(); + let chart = serde_yaml::Value::deserialize(document).unwrap(); + let chart = serde_json::to_value(chart).unwrap(); + let generated = serde_json::to_value(KarsSandbox::crd()).unwrap(); + let path = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/credentialsRef"; + for key in ["name", "uid"] { + for attribute in ["type", "minLength", "maxLength", "pattern"] { + assert_eq!( + chart.pointer(path).unwrap()["properties"][key][attribute], + generated.pointer(path).unwrap()["properties"][key][attribute], + "{key}/{attribute}" + ); + } + } +} + +#[tokio::test] +async fn every_wired_managed_runtime_accepts_the_same_agent_projection() { + for (kind, variant, config) in [ + ("OpenClaw", "openclaw", json!({})), + ("OpenAIAgents", "openaiAgents", json!({})), + ( + "MicrosoftAgentFramework", + "microsoftAgentFramework", + json!({"language": "python"}), + ), + ("LangGraph", "langGraph", json!({"language": "python"})), + ("LangGraph", "langGraph", json!({"language": "typescript"})), + ("Anthropic", "anthropic", json!({})), + ("PydanticAi", "pydanticAi", json!({})), + ("Hermes", "hermes", json!({})), + ( + "BYO", + "byo", + json!({"image": "example.test/agent:latest", "contractVersion": "v1"}), + ), + ] { + let mut state = State::new(); + let mut runtime = json!({"kind": kind}); + runtime[variant] = config; + state.sandbox.spec.runtime = serde_json::from_value(runtime).unwrap(); + let sandbox = state.sandbox.clone(); + let (_server, client, _) = setup(state).await; + let mode = reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert_eq!( + mode.env_from("demo"), + json!([ + {"secretRef": {"name": "demo-credential-projection", "optional": false}} + ]), + "{kind}" + ); + } +} + +#[tokio::test] +async fn empty_source_removes_all_owned_keys_and_does_not_cause_repeated_rollouts() { + let (server, client, state) = setup(State::new()).await; + let first = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &first); + { + let mut state = state.lock().unwrap(); + state.source.as_mut().unwrap().data = None; + state.source.as_mut().unwrap().metadata.resource_version = Some("empty-source".into()); + } + let empty = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert!( + state + .lock() + .unwrap() + .projection + .as_ref() + .unwrap() + .data + .as_ref() + .is_none_or(BTreeMap::is_empty) + ); + resume(&state, &empty); + let before = server.received_requests().await.unwrap().len(); + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert!( + server.received_requests().await.unwrap()[before..] + .iter() + .all(|request| request.method == "GET") + ); +} + +#[tokio::test] +async fn unsafe_reference_names_are_rejected_without_reading_the_named_secret() { + for name in [ + "controller-receipt-identity", + "../other", + "kars-credential-source-other", + ] { + let mut state = State::new(); + state.sandbox.spec.credentials_ref.as_mut().unwrap().name = name.into(); + let sandbox = state.sandbox.clone(); + let (server, client, _) = setup(state).await; + assert!( + reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(|request| request.url.path() == SOURCE_PATH) + ); + } +} + +#[tokio::test] +async fn opaque_type_and_explicit_runtime_env_conflicts_fail_closed_before_projection() { + for variant in ["token-type", "runtime-override", "overlay"] { + let mut state = State::new(); + match variant { + "token-type" => { + state.source.as_mut().unwrap().type_ = + Some("kubernetes.io/service-account-token".into()) + } + "runtime-override" => { + state + .sandbox + .spec + .runtime + .openclaw + .as_mut() + .unwrap() + .extra_env = Some(BTreeMap::from([( + "TELEGRAM_BOT_TOKEN".into(), + "old-value".into(), + )])); + } + _ => { + state.sandbox.spec.upstream_compatibility = Some( + serde_json::from_value(json!({ + "sigsAgentSandbox": "overlay", "upstreamSandboxName": "external" + })) + .unwrap(), + ) + } + } + let sandbox = state.sandbox.clone(); + let (server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert!(state.lock().unwrap().projection.is_none()); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(nonempty_value_write) + ); + } +} + +#[test] +fn extracted_environment_merge_retains_existing_reserved_prefix_and_precedence_rules() { + let runtime = serde_json::from_value(json!({ + "kind": "OpenClaw", "openclaw": {"extraEnv": { + "AGT_KEY": "blocked", "AZURE_CLIENT_SECRET": "blocked", "KARS_TOKEN": "blocked", + "EXISTING": "ignored", "SAFE": "allowed", "NUL": "bad\u{0000}value", "1BAD": "invalid" + }} + })) + .unwrap(); + let mut plan = super::super::runtime::build_runtime_plan(&runtime, "agent:latest").unwrap(); + plan.raw_env = vec![ + json!({"name": "AGT_OTHER", "value": "blocked"}), + json!({"name": "SAFE", "value": "ignored"}), + json!({"name": "FROM_SECRET", "valueFrom": {"secretKeyRef": {"name": "owned", "key": "key"}}}), + ]; + let mut env = vec![json!({"name": "EXISTING", "value": "controller"})]; + super::super::agent_env::merge(&mut env, &plan); + assert_eq!( + env, + vec![ + json!({"name": "EXISTING", "value": "controller"}), + json!({"name": "SAFE", "value": "allowed"}), + json!({"name": "FROM_SECRET", "valueFrom": {"secretKeyRef": {"name": "owned", "key": "key"}}}), + ] + ); +} + +#[test] +fn credential_status_acknowledges_metadata_versions_without_containing_values() { + let mode = Mode::Source { + uid: "projection".into(), + version: "100".into(), + source_uid: "source-a".into(), + source_version: "101".into(), + }; + let mut sandbox = sandbox(); + assert!(mode.needs_status_update(&sandbox)); + let condition = mode.condition(&sandbox).unwrap(); + assert_eq!(condition.type_, "CredentialsReady"); + assert!(condition.message.contains("sourceVersion")); + assert!(!condition.message.contains("initial")); + sandbox.status = Some(crate::crd::KarsSandboxStatus { + conditions: vec![condition], + ..Default::default() + }); + assert!(!mode.needs_status_update(&sandbox)); + assert!(Mode::Legacy.needs_status_update(&sandbox)); +} + +#[tokio::test] +async fn a_recreated_projection_cannot_inherit_the_previous_uid_seal() { + let (_server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &mode); + let foreign = { + let mut state = state.lock().unwrap(); + state.projection.as_mut().unwrap().metadata.uid = Some("replacement-uid".into()); + state.projection.clone() + }; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert_eq!(state.lock().unwrap().projection, foreign); + assert_eq!( + state + .lock() + .unwrap() + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); +} + +#[tokio::test] +async fn an_empty_controller_anchor_recovers_after_a_failed_uid_seal() { + let mut state = State::new(); + state.fault = Fault::FailSealOnce; + let (_server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert_eq!(state.lock().unwrap().successful_value_writes, 0); + let current = state.lock().unwrap().sandbox.clone(); + reconcile(&client, ¤t, Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert_eq!(state.lock().unwrap().successful_value_writes, 1); +} diff --git a/controller/src/reconciler/credential_source_workloads.rs b/controller/src/reconciler/credential_source_workloads.rs new file mode 100644 index 00000000..ecf44ac4 --- /dev/null +++ b/controller/src/reconciler/credential_source_workloads.rs @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; + +fn consumer(meta: &ObjectMeta, sandbox: &KarsSandbox, ns: &Namespace) -> bool { + annotation(meta, SANDBOX_UID) == sandbox.metadata.uid.as_deref() + && annotation(meta, NAMESPACE_UID) == ns.metadata.uid.as_deref() +} + +fn owned(meta: &ObjectMeta, sandbox: &KarsSandbox, ns: &Namespace) -> Result<(), Error> { + identity(meta)?; + let labels = meta.labels.as_ref().cloned().unwrap_or_default(); + let authored = meta.managed_fields.as_ref().is_some_and(|fields| { + fields.iter().any(|entry| { + entry.manager.as_deref() == Some(crate::field_managers::CLAWSANDBOX) + && entry.operation.as_deref() == Some("Apply") + && entry + .fields_v1 + .as_ref() + .is_some_and(|fields| fields.0.get("f:spec").is_some()) + }) + }); + if meta.name != sandbox.metadata.name + || meta.namespace.as_deref() != Some(ns.name_any().as_str()) + || meta.deletion_timestamp.is_some() + || meta + .owner_references + .as_ref() + .is_some_and(|refs| !refs.is_empty()) + || labels.get("kars.azure.com/sandbox") != sandbox.metadata.name.as_ref() + || labels.get("kars.azure.com/component").map(String::as_str) != Some("sandbox") + || labels + .get("kars.azure.com/parent-namespace") + .is_some_and(|value| Some(value) != sandbox.metadata.namespace.as_ref()) + || (!authored && !consumer(meta, sandbox, ns)) + { + return Err(Error::Invalid( + "runtime Deployment ownership is unproven; no workload takeover", + )); + } + Ok(()) +} + +pub(super) async fn pause( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + only_consumer: bool, +) -> Result<(), Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let deployment = match api.get(&sandbox.name_any()).await { + Ok(value) => value, + Err(kube::Error::Api(status)) if status.code == 404 => return Ok(()), + Err(error) => return Err(api_error("read runtime Deployment", error)), + }; + if only_consumer && !consumer(&deployment.metadata, sandbox, ns) { + return Ok(()); + } + owned(&deployment.metadata, sandbox, ns)?; + let strategy = if sandbox.spec.credentials_ref.is_some() { + "Recreate" + } else { + "RollingUpdate" + }; + if deployment.spec.as_ref().and_then(|spec| spec.replicas) == Some(0) + && deployment + .spec + .as_ref() + .and_then(|spec| spec.strategy.as_ref()) + .is_some_and(|value| { + value.type_.as_deref() == Some(strategy) + && (strategy != "Recreate" || value.rolling_update.is_none()) + }) + { + return Ok(()); + } + namespace_current(client, sandbox, ns).await?; + let (uid, rv) = identity(&deployment.metadata)?; + api.patch( + &sandbox.name_any(), + &PatchParams::default(), + &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv}, + "spec": {"replicas": 0, "strategy": {"type": strategy, "rollingUpdate": null}} + })), + ) + .await + .map_err(|e| api_error("stop credential consumer", e))?; + Ok(()) +} + +pub(super) async fn current( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + mode: &Mode, +) -> Result { + let Mode::Source { uid, version, .. } = mode else { + return Ok(false); + }; + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let deployment = match api.get(&sandbox.name_any()).await { + Ok(value) => value, + Err(kube::Error::Api(status)) if status.code == 404 => return Ok(false), + Err(error) => return Err(api_error("read credential consumer", error)), + }; + owned(&deployment.metadata, sandbox, ns)?; + let expected = format!("{uid}:{version}"); + Ok(consumer(&deployment.metadata, sandbox, ns) + && deployment + .spec + .as_ref() + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|meta| annotation(meta, POD_VERSION)) + == Some(expected.as_str())) +} diff --git a/controller/src/reconciler/credential_sources.rs b/controller/src/reconciler/credential_sources.rs new file mode 100644 index 00000000..e6099c53 --- /dev/null +++ b/controller/src/reconciler/credential_sources.rs @@ -0,0 +1,509 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Exclusive, opt-in agent credentials. Values only enter a UID/RV-fenced +//! projection after source, Sandbox, namespace, and target identity checks. + +use crate::{crd::KarsSandbox, credential_source::*}; +use k8s_openapi::{ + ByteString, + api::{ + apps::v1::{Deployment, DeploymentStrategy}, + core::v1::{Namespace, Secret}, + }, + apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}, +}; +use kube::{ + Api, Client, ResourceExt, + api::{Patch, PatchParams, PostParams}, + core::PartialObjectMeta, +}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[path = "credential_source_projection.rs"] +mod projection; +#[path = "credential_source_workloads.rs"] +mod workloads; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("CredentialSourceUnavailable: {0}")] + Invalid(&'static str), + #[error("CredentialSourceUnavailable: {stage} failed (Kubernetes status {code:?})")] + Api { + stage: &'static str, + code: Option, + }, +} + +fn api_error(stage: &'static str, error: kube::Error) -> Error { + Error::Api { + stage, + code: match error { + kube::Error::Api(status) => Some(status.code), + _ => None, + }, + } +} + +fn identity(meta: &ObjectMeta) -> Result<(&str, &str), Error> { + match (meta.uid.as_deref(), meta.resource_version.as_deref()) { + (Some(uid), Some(rv)) if !uid.is_empty() && !rv.is_empty() => Ok((uid, rv)), + _ => Err(Error::Invalid("API object omitted UID/resourceVersion")), + } +} + +fn annotation<'a>(meta: &'a ObjectMeta, key: &str) -> Option<&'a str> { + meta.annotations.as_ref()?.get(key).map(String::as_str) +} + +async fn metadata( + api: &Api, + name: &str, +) -> Result>, Error> { + match api.get_metadata(name).await { + Ok(value) => Ok(Some(value)), + Err(kube::Error::Api(status)) if status.code == 404 => Ok(None), + Err(error) => Err(api_error("read Secret metadata", error)), + } +} + +async fn namespace_current( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, +) -> Result<(), Error> { + super::namespace_ownership::recheck(client, sandbox, ns) + .await + .map_err(|_| Error::Invalid("runtime namespace authority changed"))?; + Ok(()) +} + +async fn sandbox_current(client: &Client, sandbox: &KarsSandbox) -> Result<(), Error> { + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let live = api + .get_metadata(&sandbox.name_any()) + .await + .map_err(|e| api_error("recheck Sandbox", e))?; + if identity(&live.metadata)? != identity(&sandbox.metadata)? + || live.metadata.namespace != sandbox.metadata.namespace + || live.metadata.name != sandbox.metadata.name + || live.metadata.deletion_timestamp.is_some() + { + return Err(Error::Invalid("Sandbox identity/reference changed")); + } + Ok(()) +} + +fn source_owner(sandbox: &KarsSandbox) -> Result { + Ok(OwnerReference { + api_version: "kars.azure.com/v1alpha1".into(), + kind: "KarsSandbox".into(), + name: sandbox.name_any(), + uid: identity(&sandbox.metadata)?.0.into(), + controller: Some(true), + block_owner_deletion: Some(false), + }) +} + +fn source_valid( + meta: &ObjectMeta, + sandbox: &KarsSandbox, + ns: &Namespace, + bound: bool, +) -> Result<(), Error> { + let reference = sandbox + .spec + .credentials_ref + .as_ref() + .ok_or(Error::Invalid("reference absent"))?; + if identity(meta)?.0 != reference.uid + || meta.name.as_deref() != Some(reference.name.as_str()) + || meta.namespace != sandbox.metadata.namespace + || meta.deletion_timestamp.is_some() + || annotation(meta, PURPOSE) != Some(SOURCE_PURPOSE) + || annotation(meta, TARGET) != sandbox.metadata.name.as_deref() + || annotation(meta, WORKSPACE) != sandbox.metadata.namespace.as_deref() + || annotation(meta, INTENT) != Some(BINDING_INTENT) + { + return Err(Error::Invalid( + "source identity, purpose, target, or lifecycle is invalid", + )); + } + let owner = source_owner(sandbox)?; + let refs = meta.owner_references.as_deref().unwrap_or_default(); + if (!refs.is_empty() && refs != [owner.clone()]) || (bound && refs != [owner]) { + return Err(Error::Invalid("source belongs to another owner")); + } + for (key, expected) in [ + (SANDBOX_UID, identity(&sandbox.metadata)?.0), + (NAMESPACE_UID, identity(&ns.metadata)?.0), + ] { + match annotation(meta, key) { + Some(value) if value == expected => {} + None if !bound => {} + _ => { + return Err(Error::Invalid( + "source Sandbox/namespace UID binding is invalid", + )); + } + } + } + Ok(()) +} + +fn validate_values(secret: &Secret) -> Result<(), Error> { + if secret.type_.as_deref() != Some("Opaque") + || secret + .string_data + .as_ref() + .is_some_and(|data| !data.is_empty()) + { + return Err(Error::Invalid("source must be a persisted Opaque Secret")); + } + let mut bytes = 0usize; + for (key, value) in secret.data.iter().flatten() { + if !AGENT_KEYS.contains(&key.as_str()) + || super::agent_env::RESERVED_PREFIXES + .iter() + .any(|prefix| key.starts_with(prefix)) + || value.0.contains(&0) + || std::str::from_utf8(&value.0).is_err() + { + return Err(Error::Invalid( + "source contains disallowed agent keys or invalid environment values", + )); + } + bytes += value.0.len(); + } + if bytes > 131_072 { + return Err(Error::Invalid( + "source credential collection exceeds 128 KiB", + )); + } + Ok(()) +} + +async fn read_source( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + default_image: &str, +) -> Result { + let reference = sandbox + .spec + .credentials_ref + .as_ref() + .ok_or(Error::Invalid("reference absent"))?; + if !valid_ref(reference, &sandbox.name_any()) { + return Err(Error::Invalid( + "reference must pin the reserved source name and exact UID", + )); + } + if sandbox + .spec + .upstream_compatibility + .as_ref() + .is_some_and(|value| value.is_overlay_mode()) + { + return Err(Error::Invalid( + "credential sources require a controller-managed runtime", + )); + } + let plan = + super::runtime::build_runtime_plan(&sandbox.spec.runtime, default_image).map_err(|_| { + Error::Invalid("credential source runtime configuration is invalid or unsupported") + })?; + if AGENT_KEYS + .iter() + .any(|key| plan.runtime_extra_env.contains_key(*key)) + || plan.raw_env.iter().any(|entry| { + entry + .get("name") + .and_then(Value::as_str) + .is_some_and(|key| AGENT_KEYS.contains(&key)) + }) + { + return Err(Error::Invalid( + "source mode cannot be combined with runtime credential env overrides", + )); + } + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let meta = metadata(&api, &reference.name) + .await? + .ok_or(Error::Invalid("source Secret is missing"))?; + source_valid(&meta.metadata, sandbox, ns, false)?; + let mut source = api + .get(&reference.name) + .await + .map_err(|e| api_error("read credential source", e))?; + if identity(&source.metadata)? != identity(&meta.metadata)? { + return Err(Error::Invalid("source changed during read")); + } + source_valid(&source.metadata, sandbox, ns, false)?; + validate_values(&source)?; + if source_valid(&source.metadata, sandbox, ns, true).is_err() { + let (uid, rv) = identity(&source.metadata)?; + let bound = api.patch_metadata(&reference.name, &PatchParams::default(), &Patch::Merge(json!({ + "metadata": { + "uid": uid, "resourceVersion": rv, + "ownerReferences": [source_owner(sandbox)?], + "annotations": {SANDBOX_UID: sandbox.metadata.uid, NAMESPACE_UID: ns.metadata.uid} + } + }))).await.map_err(|e| api_error("bind credential source", e))?; + source.metadata = bound.metadata; + source_valid(&source.metadata, sandbox, ns, true)?; + } + Ok(source) +} + +async fn inputs_current( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + source: &Secret, +) -> Result<(), Error> { + sandbox_current(client, sandbox).await?; + namespace_current(client, sandbox, ns).await?; + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let live = metadata(&api, &source.name_any()) + .await? + .ok_or(Error::Invalid("source disappeared"))?; + source_valid(&live.metadata, sandbox, ns, true)?; + if identity(&live.metadata)? != identity(&source.metadata)? { + return Err(Error::Invalid("source changed before projection write")); + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub enum Mode { + Legacy, + Source { + uid: String, + version: String, + source_uid: String, + source_version: String, + }, +} + +impl Mode { + pub fn env_from(&self, sandbox: &str) -> Value { + match self { + Self::Legacy => { + json!([{"secretRef": {"name": format!("{sandbox}-credentials"), "optional": true}}]) + } + Self::Source { .. } => { + json!([{"secretRef": {"name": projection_name(sandbox), "optional": false}}]) + } + } + } + + pub fn decorate(&self, deployment: &mut Deployment, sandbox: &KarsSandbox, ns: &Namespace) { + if let Self::Source { uid, version, .. } = self { + let annotations = deployment.metadata.annotations.get_or_insert_default(); + annotations.insert( + SANDBOX_UID.into(), + sandbox.metadata.uid.clone().unwrap_or_default(), + ); + annotations.insert( + NAMESPACE_UID.into(), + ns.metadata.uid.clone().unwrap_or_default(), + ); + let spec = deployment + .spec + .as_mut() + .expect("controller builds a Deployment spec"); + spec.strategy = Some(DeploymentStrategy { + type_: Some("Recreate".into()), + rolling_update: None, + }); + spec.template + .metadata + .get_or_insert_default() + .annotations + .get_or_insert_default() + .insert(POD_VERSION.into(), format!("{uid}:{version}")); + } + } + + pub fn condition( + &self, + sandbox: &KarsSandbox, + ) -> Option { + let prior = sandbox.status.as_ref().and_then(|status| { + status + .conditions + .iter() + .find(|condition| condition.type_ == "CredentialsReady") + }); + let (status, reason, message) = match self { + Self::Source { + uid, + version, + source_uid, + source_version, + } => ( + "True", + "Projected", + json!({"sourceUid": source_uid, "sourceVersion": source_version, + "projectionUid": uid, "projectionVersion": version}) + .to_string(), + ), + Self::Legacy if prior.is_some() => ( + "False", + "DirectCredentials", + "Explicit source is not configured".into(), + ), + Self::Legacy => return None, + }; + Some(crate::status::conditions::preserve_transition_time( + prior, + "CredentialsReady", + status, + reason, + &message, + sandbox.metadata.generation, + )) + } + + pub fn needs_status_update(&self, sandbox: &KarsSandbox) -> bool { + self.condition(sandbox).is_some_and(|expected| { + !sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|current| { + current.type_ == expected.type_ + && current.status == expected.status + && current.reason == expected.reason + && current.message == expected.message + }) + }) + }) + } +} + +pub async fn reconcile( + client: &Client, + sandbox: &KarsSandbox, + ns: Option<&Namespace>, + default_image: &str, +) -> Result { + let ns = ns.ok_or(Error::Invalid("runtime namespace is not verified"))?; + let result = if sandbox.spec.credentials_ref.is_some() { + project(client, sandbox, ns, default_image).await + } else { + projection::detach(client, sandbox, ns) + .await + .map(|()| Mode::Legacy) + }; + if let Err(error) = result { + // Try both operations: a transient Deployment error must not skip + // projection revocation, or vice versa. Never echo API request bodies. + let stopped = + workloads::pause(client, sandbox, ns, sandbox.spec.credentials_ref.is_none()).await; + let revoked = projection::revoke(client, sandbox, ns, false).await; + let failure = stopped.err().or_else(|| revoked.err()).unwrap_or(error); + report(client, sandbox, &failure).await?; + return Err(failure); + } + result +} + +async fn project( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + default_image: &str, +) -> Result { + namespace_current(client, sandbox, ns).await?; + let source = read_source(client, sandbox, ns, default_image).await?; + let target = projection::anchor(client, sandbox, ns, &source).await?; + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let current = api + .get(&target.name_any()) + .await + .map_err(|e| api_error("read owned projection", e))?; + projection::validate(¤t.metadata, sandbox, ns)?; + if identity(¤t.metadata)? != identity(&target.metadata)? + || current.type_.as_deref() != Some("Opaque") + || current.immutable == Some(true) + { + return Err(Error::Invalid("projection identity/type changed")); + } + let values: BTreeMap = source.data.clone().unwrap_or_default(); + let changed = current.data.clone().unwrap_or_default() != values + || annotation(¤t.metadata, SOURCE_UID) != source.metadata.uid.as_deref(); + let mut mode = Mode::Source { + uid: identity(¤t.metadata)?.0.into(), + version: identity(¤t.metadata)?.1.into(), + source_uid: identity(&source.metadata)?.0.into(), + source_version: identity(&source.metadata)?.1.into(), + }; + if changed || !workloads::current(client, sandbox, ns, &mode).await? { + workloads::pause(client, sandbox, ns, false).await?; + } + inputs_current(client, sandbox, ns, &source).await?; + if changed { + let (uid, rv) = identity(¤t.metadata)?; + let mut data = serde_json::to_value(&values) + .map_err(|_| Error::Invalid("credential serialization failed"))?; + for key in current.data.iter().flat_map(|data| data.keys()) { + if !values.contains_key(key) { + data[key] = Value::Null; + } + } + let written = api.patch_metadata(&target.name_any(), &PatchParams::default(), &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv, "annotations": {SOURCE_UID: source.metadata.uid}}, + "data": data + }))).await.map_err(|e| api_error("write credential projection", e))?; + projection::validate(&written.metadata, sandbox, ns)?; + mode = Mode::Source { + uid: identity(&written.metadata)?.0.into(), + version: identity(&written.metadata)?.1.into(), + source_uid: identity(&source.metadata)?.0.into(), + source_version: identity(&source.metadata)?.1.into(), + }; + } + Ok(mode) +} + +async fn report(client: &Client, sandbox: &KarsSandbox, error: &Error) -> Result<(), Error> { + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let live = api + .get(&sandbox.name_any()) + .await + .map_err(|e| api_error("read credential status target", e))?; + if live.metadata.uid != sandbox.metadata.uid { + return Err(Error::Invalid("Sandbox was recreated")); + } + let message = error.to_string(); + let mut patch = + crate::status::build_degraded_status_patch(&live, "CredentialSourceUnavailable", &message); + let existing = serde_json::to_value(&live.status) + .map_err(|_| Error::Invalid("status serialization failed"))?; + if patch["status"].as_object().is_some_and(|fields| { + fields + .iter() + .all(|(key, val)| existing.get(key) == Some(val)) + }) { + return Ok(()); + } + let (uid, rv) = identity(&live.metadata)?; + patch["metadata"] = json!({"uid": uid, "resourceVersion": rv}); + api.patch_status( + &live.name_any(), + &PatchParams::default(), + &Patch::Merge(patch), + ) + .await + .map_err(|e| api_error("report credential failure", e))?; + Ok(()) +} + +#[cfg(test)] +#[path = "credential_source_tests.rs"] +mod tests; diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 38bcdfcd..8e86917f 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -33,7 +33,9 @@ use tokio::time::Duration; use crate::crd::KarsSandbox; use crate::fedcred::{FedCredConfig, FedCredManager}; +mod agent_env; pub(crate) mod byo_contract; +mod credential_sources; mod dev_env; pub(crate) mod governance_mounts; mod inference; @@ -60,6 +62,8 @@ fn sandbox_node_selector(default_pool: &str) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result = openclaw_env - .iter() - .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) - .collect(); - for (k, v) in &runtime_plan.runtime_extra_env { - if k.is_empty() - || !k.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - || k.chars().next().is_some_and(|c| c.is_ascii_digit()) - { - tracing::warn!(key = %k, "extraEnv: invalid env var name, skipping"); - continue; - } - if RESERVED_PREFIXES.iter().any(|p| k.starts_with(p)) { - tracing::warn!(key = %k, "extraEnv: key uses reserved prefix, skipping"); - continue; - } - if v.contains('\0') { - tracing::warn!(key = %k, "extraEnv: value contains NUL byte, skipping"); - continue; - } - if existing.contains(k) { - tracing::debug!(key = %k, "extraEnv: overridden by reconciler, skipping"); - continue; - } - openclaw_env.push(json!({"name": k, "value": v})); - existing.insert(k.clone()); - } - } - - // S10.A2.b: append `plan.raw_env` entries (BYO `valueFrom` etc.). - // The producer guarantees these have a `name` field; we apply the - // same reserved-prefix / NUL / dup filter to the `name` only — - // the `valueFrom` payload itself is rendered verbatim. - if !runtime_plan.raw_env.is_empty() { - const RESERVED_PREFIXES: &[&str] = - &["AGT_", "FOUNDRY_AGENT_", "AZURE_", "IMDS_", "KARS_"]; - let mut existing: std::collections::HashSet = openclaw_env - .iter() - .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) - .collect(); - for entry in &runtime_plan.raw_env { - let Some(name) = entry.get("name").and_then(|n| n.as_str()) else { - tracing::warn!("rawEnv: entry missing `name`, skipping"); - continue; - }; - if name.is_empty() - || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - || name.chars().next().is_some_and(|c| c.is_ascii_digit()) - { - tracing::warn!(key = %name, "rawEnv: invalid env var name, skipping"); - continue; - } - if RESERVED_PREFIXES.iter().any(|p| name.starts_with(p)) { - tracing::warn!(key = %name, "rawEnv: key uses reserved prefix, skipping"); - continue; - } - if existing.contains(name) { - tracing::debug!(key = %name, "rawEnv: overridden by reconciler, skipping"); - continue; - } - openclaw_env.push(entry.clone()); - existing.insert(name.to_string()); - } - } + agent_env::merge(&mut openclaw_env, &runtime_plan); // Build the inference-router env array let mut router_env = vec![ @@ -1929,9 +1872,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result = allowlist_resolution.conditions.clone(); + if let Some(condition) = credentials.condition(&sandbox) { + extras.push(condition); + } // Phase G P1 #4: stamp Suspended condition when spec.suspended // is true, or surface Suspended=False/Active when there is a @@ -3140,7 +3087,8 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); let status_obj = crate::status::build_running_status_patch_with_extras( @@ -3173,7 +3121,13 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result Duration { let base = match error { // Transient kube API errors (throttling, connection reset, 5xx): // retry soon so we don't starve legitimate work. - ReconcileError::Kube(_) | ReconcileError::NamespaceOwnership(_) => 30, + ReconcileError::Kube(_) + | ReconcileError::NamespaceOwnership(_) + | ReconcileError::Credentials(_) => 30, // Serde errors are deterministic — the same body will fail again. // Back off longer so we don't spam logs while a human fixes the // bad CR. @@ -3200,6 +3156,7 @@ fn error_policy(sandbox: Arc, error: &ReconcileError, _ctx: Arc "serde", ReconcileError::Configuration(_) => "configuration", ReconcileError::NamespaceOwnership(_) => "namespace_ownership", + ReconcileError::Credentials(_) => "credentials", }; crate::metrics::record_reconcile_error("KarsSandbox", class); tracing::error!( @@ -3377,6 +3334,10 @@ pub async fn run(client: Client) -> Result<()> { }); Controller::new(sandboxes, crate::watch_config::bounded()) + .owns( + Api::::all(ctx.client.clone()), + crate::watch_config::bounded(), + ) .watches( Api::::all(ctx.client.clone()), crate::watch_config::bounded(), diff --git a/deploy/helm/kars/templates/crd.yaml b/deploy/helm/kars/templates/crd.yaml index 3e9d7506..1b4e3cad 100644 --- a/deploy/helm/kars/templates/crd.yaml +++ b/deploy/helm/kars/templates/crd.yaml @@ -15,11 +15,16 @@ spec: schema: openAPIV3Schema: type: object + x-kubernetes-validations: + - rule: "!has(self.spec) || !has(self.spec.credentialsRef) || self.spec.credentialsRef.name == 'kars-credential-source-' + self.metadata.name" + message: "credentialsRef must use the reserved source name for this Sandbox" properties: spec: type: object required: ["runtime", "sandbox", "inferenceRef"] x-kubernetes-validations: + - rule: "!has(self.credentialsRef) || !has(self.upstreamCompatibility) || !has(self.upstreamCompatibility.sigsAgentSandbox) || self.upstreamCompatibility.sigsAgentSandbox != 'overlay'" + message: "credentialsRef requires a controller-managed runtime, not overlay mode" # P2 #13: top-level cross-field invariants. # # 1. BYO runtime is mutually exclusive with the @@ -45,6 +50,21 @@ spec: message: "spec.governance.toolPolicyRef.name must be a same-namespace name (no '/' or ':' separators); cross-namespace refs are forbidden" reason: "FieldValueInvalid" properties: + credentialsRef: + type: object + description: "Explicit agent credential collection in this Sandbox's workspace; replaces legacy credentials while set. Missing/replaced sources fail closed." + required: ["name", "uid"] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + pattern: "^kars-credential-source-[a-z0-9][a-z0-9-]*$" + uid: + type: string + minLength: 1 + maxLength: 128 + pattern: "^[A-Za-z0-9-]+$" runtime: type: object required: ["kind"] diff --git a/docs/how-to/credential-sources.md b/docs/how-to/credential-sources.md new file mode 100644 index 00000000..4335caec --- /dev/null +++ b/docs/how-to/credential-sources.md @@ -0,0 +1,182 @@ +# Workspace credential sources (v1) + +Credential sources are an **optional, explicit** alternative to the existing +`kars-/-credentials` Secret. Install the controller and CRD containing +this feature before enabling it. Namespace claim v1 is a prerequisite; complete +[namespace ownership preflight/adoption](namespace-ownership.md) first. + +With `spec.credentialsRef` absent, agents retain the existing direct Secret +EnvFrom and defaults. No source is discovered or delivered merely because its +name matches a future Sandbox. + +## CLI workflows + +New Sandbox, with credentials stored through the existing masked local prompt: + +```sh +kars credentials set telegram-token +kars add demo --channels telegram --credential-source +``` + +The CLI creates the workspace source **before** the Sandbox, obtains its real +API-server UID, and includes that UID in the Sandbox CREATE. A create conflict +does not overwrite a racing Sandbox. `--namespace ` selects the +workspace for the source, Sandbox, and companion policies; the default remains +`kars-system`. + +For an existing Sandbox: + +```sh +kars credentials update demo --use-source +kars credentials update demo --telegram-token "$NEW_TOKEN" +kars credentials update demo --remove telegram-token +``` + +`--use-source` migrates the entire direct credential collection once, without +changing the legacy Secret. Explicit updates override imported values. Existing +staged source values override legacy values. Removal is applied last. Migration +requires read access to the verified runtime namespace and its direct Secret; +new source-bound Sandboxes do not require runtime-namespace Secret writes. + +Once bound, ordinary `credentials update` automatically updates the pinned source +in the selected workspace. Pass `--namespace` for Sandboxes outside `kars-system`. +The command requires read access to that Sandbox to choose the correct path. +Direct updates also inspect runtime namespace claim metadata when present, so a +wrong workspace or same-name conflict cannot be mistaken for a direct target. +Source updates and removals use UID/resourceVersion checks and never +delete/recreate the source. `--remove` accepts comma-separated environment keys +or credential flag names. The existing `credentials remove ` command still +removes a **local** stored credential. + +Source mode always refreshes the runtime. `--no-restart` remains available for +direct credentials but is rejected for source changes. Source-mode operations +wait for the controller to acknowledge the current source UID/resourceVersion +in `CredentialsReady`; they do not claim success merely because an older +controller accepted a CR. API schema pruning is detected after submission. +Existing secret-value flags retain their usual shell/process-argument exposure; +the source integration passes Kubernetes Secret bodies through stdin and does +not print their values. + +Explicit opt-out: + +```sh +kars credentials update demo --disable-source +``` + +The controller stops the previous credential consumer, removes **only its owned +projection**, and returns to the unchanged direct Secret. This intentionally +reactivates the legacy collection: review it before opting out. The workspace +source remains available for the same Sandbox UID; it is not Helm-owned. + +Explicit pre-CR storage is supported: + +```sh +kars credentials update demo --use-source --telegram-token "$TOKEN" +kars add demo --credential-source +``` + +The first command stores an opted-in source, not a delivery instruction. The +second explicitly selects its UID. Abandoned unbound sources can have their +keys removed with `credentials update --use-source --remove`; normal Kubernetes +Secret deletion can remove the empty resource. Bound sources have a same- +workspace Sandbox owner reference and are garbage-collected with that Sandbox. +A different Sandbox UID cannot inherit a previous binding. + +## API contract + +The source is an Opaque Secret in the **same namespace as the KarsSandbox CR**: + +* Name: `kars-credential-source-`. +* Annotations: + * `kars.azure.com/credential-purpose: agent-source-v1` + * `kars.azure.com/credential-target: ` + * `kars.azure.com/credential-workspace: ` + * `kars.azure.com/credential-binding-intent: explicit-reference-v1` +* Data: the complete desired agent credential collection. + +After creating it, set: + +```yaml +spec: + credentialsRef: + name: kars-credential-source-demo + uid: +``` + +There is no namespace field in the reference. The controller rejects arbitrary +source names, missing purpose/intent, different target/workspace, different UID, +non-Opaque type, foreign owner references, or stale bindings. Initial binding +records the exact Sandbox UID and runtime namespace UID, with a UID/RV-fenced +metadata-only patch. Existing binding metadata must agree; it is not overwritten +to claim another Sandbox's source. + +An intentionally replaced source requires a new explicit reference. +`credentials update --use-source` selects a new **compatible** source incarnation; +ordinary updates refuse UID drift. Foreign purpose/ownership is never adopted, +even with `--use-source`. + +## Collection semantics and supported keys + +While a reference is present, **only the owned projection is mounted as the +agent credential collection**. The direct Secret is not a fallback. Therefore +removing a source key cannot resurrect an old same-named legacy key. + +Version 1 uses the existing handoff channel/search credential allowlist: + +`TELEGRAM_BOT_TOKEN`, `TELEGRAM_ALLOW_FROM`, `SLACK_BOT_TOKEN`, +`DISCORD_BOT_TOKEN`, `WHATSAPP_ENABLED`, `BRAVE_API_KEY`, `TAVILY_API_KEY`, +`EXA_API_KEY`, `FIRECRAWL_API_KEY`, `PERPLEXITY_API_KEY`. + +Values must be UTF-8 without NUL; the collection is limited to 128 KiB. +Router/provider/control-plane and arbitrary process environment keys are not +accepted. In particular, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, Azure identity +keys, `AGT_*`, `KARS_*`, `NODE_OPTIONS`, and `PATH` cannot be supplied this way. +Existing direct-mode flags, including `--openai-api-key`, remain available. +Migration fails rather than silently dropping unsupported existing keys. + +Credential-key overrides in runtime `extraEnv`/raw `env` conflict with source +mode and are rejected, rather than silently overriding the source. The common +agent container path supports every **wired, controller-managed** runtime, +including BYO. The projection is never added to the inference-router EnvFrom. +Overlay-managed pods and unwired runtime variants are not supported. + +## Projection, refresh, and failure handling + +The controller owns `-credential-projection` in the verified +runtime namespace. Its metadata binds purpose, workspace, Sandbox UID, namespace +UID, and source UID, with a namespace owner reference. Unrelated Secrets at that +name are not overwritten or adopted. + +An empty metadata-only anchor is created first and sealed to its own Secret UID. +Recreating an object with copied old projection metadata does not authorize +adoption of the new UID. The controller rechecks the +Sandbox UID/RV, namespace incarnation, and source UID/RV before a UID/RV-fenced +value patch. Removed keys use explicit nulls. No force-apply is used for the +projection. Credential values and API bodies are excluded from errors/logs. + +Source-owned Secrets are watched through kube's **metadata-only ownership +watch**. A 30-second reconciliation backstop covers deletion, removed ownership +markers, unbound/invalid sources, and projection drift. Projection UID/RV drives +the pod-template revision; metadata-only source changes do not restart pods. + +Source mode uses `Recreate`, not a rolling update that could indefinitely retain +an old credential-bearing pod when a replacement fails. Changes pause the +verified controller Deployment before refresh. Invalid, missing, or replaced +references stop that runtime and clear only the owned projection, then report +`CredentialSourceUnavailable` and retry. They never fall back to direct +credentials. Concurrent writes/replacements and non-404 API failures propagate +honestly; retries restart with fresh identities. + +Kubernetes is not a cross-resource transaction system. Watches/requeues make +revocation asynchronous; API outages, unreachable nodes, and pod termination +failures can delay it. This feature does not revoke credentials at their external +provider, erase values previously observed by an agent, or defend against a +cluster administrator deliberately bypassing namespace lifecycle controls. + +## Bridge and publication boundary + +This is a standalone core primitive, **not closure of Bridge RBAC**. A later +adapter must store the source in its configured workspace and include the exact +UID in launch bindings. Existing Bridge `put_credential` behavior is unchanged. +Workspace Secret permissions/admission must still protect signing and other +control-plane Secrets; no new broad Role or binding is introduced here. diff --git a/docs/how-to/helm-installation.md b/docs/how-to/helm-installation.md index 70ba0ea2..f72c1e6c 100644 --- a/docs/how-to/helm-installation.md +++ b/docs/how-to/helm-installation.md @@ -9,6 +9,10 @@ updated CLI and resolve all ownership conflicts. See [namespace ownership migration and adoption](namespace-ownership.md). The check does not read Secrets or change namespaces, workloads, or Helm values. +Optional [workspace credential sources](credential-sources.md) require the +matching controller and CRD. Upgrade both before using `--credential-source`; +legacy direct credential Secrets remain the default. + ## Local kind ```bash diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md new file mode 100644 index 00000000..1c1b71a9 --- /dev/null +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -0,0 +1,96 @@ +# Agent credential-source capability review — 2026-09-07 + +**Status:** additive candidate; independent security review and human sign-off +pending. Automated tests are evidence, not approval. No reviewer identity, +organizational approval, compliance certification, or production qualification +is asserted by this document. + +## Scope and base + +Base: namespace prerequisite `62093414cb8d5d9937c1d9974504047c84669d6c` +(Azure/kars#548). This slice adds an optional UID-pinned KarsSandbox credential +source, controller-owned projection, CLI source lifecycle, and schema/docs. +It does not change Bridge permissions, inference credentials, signing providers, +agent transport, or customer/live deployments. + +The optional field is omitted from serialization when unset, preserving old +Sandbox-spec inputs. When present, its name/UID remain in serialized spec +digests. Existing Task/Team blueprint and receipt formats are not extended: +mutable credential values are not claimed to be covered by a governance receipt. + +## Authority boundaries + +| Boundary | Control | +|---|---| +| Arbitrary Secret read oracle | Fixed source-name derivation; same CR workspace; exact UID; explicit purpose/intent/target before reading values | +| Source type and data | Opaque only; existing agent channel/search allowlist; provider/control-plane/process keys rejected | +| Sandbox recreation | Exact source owner binding to Sandbox UID; no automatic name-only future delivery | +| Workspace/name collisions | Namespace claim v1 and runtime namespace UID rechecks | +| Foreign projection | Exact projection purpose, namespace owner reference, Sandbox/workspace/namespace UID bindings | +| Cross-resource write races | Empty anchor, source/Sandbox/namespace rechecks, UID/RV-fenced final value patch; no force takeover | +| Rotation/removal | Exclusive collection, explicit removed-key nulls, UID/RV revision, controlled stop/Recreate | +| Source invalidation | Stop verified runtime, revoke only owned projection, report failure, bounded retry; no legacy fallback | +| Error disclosure | Kubernetes API errors reduced to stage/status; no request bodies or credential values in source errors | +| CLI authority | Source before CR CREATE; exact returned UID; source update CAS; explicit UID reselection; schema-retention and controller-version acknowledgement checks | + +Unconfigured Sandboxes keep their direct EnvFrom collection. Explicit opt-in +performs a one-time CLI migration without editing the direct Secret. Explicit +opt-out restores that direct collection. Neither operation may silently move +foreign or provider/control-plane data through the source primitive. + +## Automated evidence + +Existing Rust/Vitest/Helm runners cover: + +* Missing-reference legacy behavior and default schema/serialization. +* Explicit source creation/binding and fenced projection writes. +* Key addition, update, removal, empty collections, and metadata-only changes. +* Missing/replaced source, wrong workspace/name, foreign owner/type/purpose, + namespace replacement, destination collision, and API/CAS races. +* Owned-only revocation and controller deployment pause/refresh. +* Source values excluded from error text and process-environment injection. +* CLI pre-CR storage, actual returned UID binding, collection migration, + update/removal/disable, safe errors, and direct-path compatibility. +* Generated/reference schema parity and Helm admission-rule rendering. + +Candidate qualification on 2026-09-07: + +* 66 targeted controller tests passed (credential-source, namespace ownership, + and SRE-writer compatibility selectors), offline/locked with incremental + compilation disabled. +* Controller all-targets Clippy passed with warnings denied. +* 90 targeted CLI/Vitest tests passed, including Helm rendering and new-module + size/header/no-stub/no-custom-crypto assertions. +* TypeScript typecheck and changed-file oxlint passed. + +The existing disposable Kind harness now includes a source-bound BYO fixture +using its already-loaded sandbox test image. The consumer emits fixed markers, +not environment values. The lifecycle covers actual process environment on +initial delivery, rotation and key removal; missing-source revocation with no +legacy fallback; explicit opt-out restoring the preserved direct collection; +and cleanup preserving the core namespace. Shell syntax is checked locally; +execution of this new case awaits hosted CI. + +Diff-based publication gates still run on the parent's eventual atomic commit; +the candidate was intentionally not committed or pushed here. No live +Kubernetes admission or customer rollout was executed. Real-cluster +qualification and independent/human review remain release gates. + +## Remaining limits and required review + +1. Kubernetes source/target operations are not transactional. Last-read identity + checks plus a fenced target write prevent blind replacement writes; watches + and a bounded retry provide eventual source revocation. +2. API failures/node partitions can delay process termination. External + credential invalidation and secrets already observed by an agent are outside + this feature's guarantees. +3. Namespace administrators remain trusted. Namespace claim annotations are not + cryptographic attestations or protection from forced namespace finalization. +4. CLI credential flags retain their existing shell/process-argument exposure. + Kubernetes source payloads themselves travel through stdin and are not logged. +5. Bridge still needs a separate reviewed adapter and least-privilege/admission + changes. This candidate must not be described as Bridge Secret-RBAC closure. +6. Review schema/controller mixed-version handling, consumer rollout behavior, + and source-authoring permissions before release. + +**Human sign-off: pending. Independent security sign-off: pending.** diff --git a/tests/e2e/credential-sources.sh b/tests/e2e/credential-sources.sh new file mode 100644 index 00000000..58c5dd71 --- /dev/null +++ b/tests/e2e/credential-sources.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Uses only the existing disposable Kind cluster and its loaded BYO test image. +# The consumer emits fixed markers, never environment values. +wait_for_credential_consumer() { + local marker="$1" deadline=$(($(date +%s) + 150)) pod namespace + local k=(kubectl --context kind-kars-e2e) + while [ "$(date +%s)" -lt "$deadline" ]; do + namespace=$("${k[@]}" get namespace kars-e2e-source --ignore-not-found \ + -o jsonpath='{.metadata.uid}') || return 1 + if [ -z "$namespace" ]; then + sleep 2 + continue + fi + pod=$("${k[@]}" get pods -n kars-e2e-source -l kars.azure.com/sandbox=e2e-source \ + -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' \ + 2>/dev/null | head -1) || return 1 + if [ -n "$pod" ] && "${k[@]}" logs "$pod" -n kars-e2e-source -c agent --tail=10 2>/dev/null \ + | grep -qx "$marker"; then + return 0 + fi + sleep 2 + done + fail "Credential consumer did not reach $marker" + return 1 +} + +test_credential_sources() { + local k=(kubectl --context kind-kars-e2e) + local source_uid projection_uid direct_uid namespace_uid projection + namespace_uid=$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}') || return 1 + if ! "${k[@]}" create -f - <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: kars-credential-source-e2e-source + namespace: kars-system + annotations: + kars.azure.com/credential-purpose: agent-source-v1 + kars.azure.com/credential-target: e2e-source + kars.azure.com/credential-workspace: kars-system + kars.azure.com/credential-binding-intent: explicit-reference-v1 +type: Opaque +stringData: + BRAVE_API_KEY: fixture-one + TAVILY_API_KEY: fixture-auxiliary +--- +apiVersion: kars.azure.com/v1alpha1 +kind: InferencePolicy +metadata: + name: e2e-source-inference + namespace: kars-system +spec: + appliesTo: + sandboxName: e2e-source + modelPreference: + primary: + provider: azure-openai + deployment: gpt-4.1 +YAML + then + fail "Cannot create the credential-source fixtures"; return 1 + fi + source_uid=$("${k[@]}" get secret kars-credential-source-e2e-source -n kars-system \ + -o jsonpath='{.metadata.uid}') || return 1 + [ -n "$source_uid" ] || { fail "Source fixture omitted its UID"; return 1; } + if ! "${k[@]}" create -f - </dev/null || return 1 + direct_uid=$("${k[@]}" get secret e2e-source-credentials -n kars-e2e-source \ + -o jsonpath='{.metadata.uid}') || return 1 + "${k[@]}" patch secret kars-credential-source-e2e-source -n kars-system --type=merge \ + -p '{"stringData":{"BRAVE_API_KEY":"fixture-two"},"data":{"TAVILY_API_KEY":null}}' \ + >/dev/null || return 1 + wait_for_credential_consumer CREDENTIAL_FIXTURE_ROTATED || return 1 + if [ "$("${k[@]}" get secret e2e-source-credential-projection -n kars-e2e-source -o jsonpath='{.metadata.uid}')" != "$projection_uid" ]; then + fail "Rotation replaced the projection instead of updating the owned Secret"; return 1 + fi + pass "Source rotation replaces the consumer and removed keys do not fall back to legacy values" + + "${k[@]}" delete secret kars-credential-source-e2e-source -n kars-system --timeout=30s >/dev/null || return 1 + local deadline=$(($(date +%s) + 150)) revoked=0 reason replicas pods keys + while [ "$(date +%s)" -lt "$deadline" ]; do + reason=$("${k[@]}" get karssandbox e2e-source -n kars-system \ + -o jsonpath='{.status.conditions[?(@.type=="Degraded")].reason}') || return 1 + replicas=$("${k[@]}" get deployment e2e-source -n kars-e2e-source \ + -o jsonpath='{.spec.replicas}') || return 1 + pods=$("${k[@]}" get pods -n kars-e2e-source -l kars.azure.com/sandbox=e2e-source \ + -o go-template='{{len .items}}') || return 1 + keys=$("${k[@]}" get secret e2e-source-credential-projection -n kars-e2e-source \ + --ignore-not-found -o go-template='{{if .data}}{{len .data}}{{else}}0{{end}}') || return 1 + if [ "$reason" = CredentialSourceUnavailable ] && [ "$replicas" = 0 ] && [ "$pods" = 0 ] \ + && { [ -z "$keys" ] || [ "$keys" = 0 ]; }; then + revoked=1 + break + fi + sleep 2 + done + if [ "$revoked" != 1 ]; then + fail "Source deletion did not stop the consumer and revoke its owned projection"; return 1 + fi + if [ "$("${k[@]}" get secret e2e-source-credentials -n kars-e2e-source -o jsonpath='{.metadata.uid}')" != "$direct_uid" ]; then + fail "Source revocation changed the unrelated legacy Secret"; return 1 + fi + pass "Source deletion revokes the consumer without reactivating or deleting legacy credentials" + + "${k[@]}" patch karssandbox e2e-source -n kars-system --type=merge \ + -p '{"spec":{"credentialsRef":null}}' >/dev/null || return 1 + wait_for_credential_consumer CREDENTIAL_FIXTURE_LEGACY || return 1 + projection=$("${k[@]}" get secret e2e-source-credential-projection -n kars-e2e-source \ + --ignore-not-found -o name) || return 1 + if [ -n "$projection" ]; then + fail "Explicit source opt-out left its owned projection behind"; return 1 + fi + pass "Explicit opt-out restores the unchanged legacy collection only when requested" + + "${k[@]}" delete karssandbox e2e-source -n kars-system --wait=false >/dev/null || return 1 + "${k[@]}" wait --for=delete namespace/kars-e2e-source --timeout=120s || return 1 + "${k[@]}" delete inferencepolicy e2e-source-inference -n kars-system >/dev/null || return 1 + if [ "$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}')" != "$namespace_uid" ]; then + fail "Credential-source cleanup changed the core namespace"; return 1 + fi + pass "Credential-source lifecycle cleanup preserves the core namespace" +} diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 34536369..4be72545 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -3015,6 +3015,7 @@ EOF # ─── Main ───────────────────────────────────────────────────────────────────── source "$SCRIPT_DIR/namespace-ownership.sh" +source "$SCRIPT_DIR/credential-sources.sh" main() { echo "" @@ -3115,6 +3116,7 @@ main() { esac test_sre_namespace_ownership || fail "SRE namespace lifecycle gate failed" + test_credential_sources || fail "Credential-source lifecycle gate failed" echo "" echo "═══════════════════════════════════════════════════════" From 693a46ca67ceb760e8be4b5681dd4765fb85c69b Mon Sep 17 00:00:00 2001 From: pallakatos Date: Mon, 7 Sep 2026 23:06:47 +0200 Subject: [PATCH 2/6] fix(helm): declare credential admission schema dependencies Declare the existing optional upstream compatibility fields used by CEL so Kubernetes can install the Sandbox CRD. Keep overlay/source validation, assert referenced schema fields, require the intended live admission rejection, and stop E2E setup on Helm failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/testing/credential-source.test.ts | 17 ++++++++++ deploy/helm/kars/templates/crd.yaml | 20 +++++++++++ .../2026-09-07-credential-sources.md | 15 ++++++++ tests/e2e/credential-sources.sh | 34 ++++++++++++++++++- tests/e2e/run.sh | 1 + 5 files changed, 86 insertions(+), 1 deletion(-) diff --git a/cli/src/testing/credential-source.test.ts b/cli/src/testing/credential-source.test.ts index 015aedfd..5045db9c 100644 --- a/cli/src/testing/credential-source.test.ts +++ b/cli/src/testing/credential-source.test.ts @@ -23,7 +23,24 @@ describe("credential source public integration", () => { const schema = resource.spec.versions[0].schema.openAPIV3Schema; const spec = schema.properties.spec; expect(spec.required).not.toContain("credentialsRef"); + expect(spec.required).not.toContain("upstreamCompatibility"); expect(spec.properties.credentialsRef.required).toEqual(["name", "uid"]); + // CEL compiles against declared schema fields, including those inside has(). + expect(spec.properties.upstreamCompatibility).toMatchObject({ + type: "object", + properties: { + sigsAgentSandbox: { type: "string", enum: ["off", "observe", "translate", "overlay"] }, + upstreamSandboxRef: { + type: "object", required: ["name"], + properties: { name: { type: "string", minLength: 1, maxLength: 253 } }, + }, + aiConformanceReference: { type: "boolean" }, + }, + }); + expect(spec.properties.upstreamCompatibility["x-kubernetes-validations"]) + .toContainEqual(expect.objectContaining({ + rule: "!has(self.sigsAgentSandbox) || self.sigsAgentSandbox != 'overlay' || has(self.upstreamSandboxRef)", + })); expect(schema["x-kubernetes-validations"].some((rule: { rule: string }) => rule.rule.includes("self.metadata.name") && rule.rule.includes("kars-credential-source-"))).toBe(true); expect(spec["x-kubernetes-validations"].some((rule: { rule: string }) => diff --git a/deploy/helm/kars/templates/crd.yaml b/deploy/helm/kars/templates/crd.yaml index 1b4e3cad..e146f7e2 100644 --- a/deploy/helm/kars/templates/crd.yaml +++ b/deploy/helm/kars/templates/crd.yaml @@ -50,6 +50,26 @@ spec: message: "spec.governance.toolPolicyRef.name must be a same-namespace name (no '/' or ':' separators); cross-namespace refs are forbidden" reason: "FieldValueInvalid" properties: + upstreamCompatibility: + type: object + description: "Existing upstream interoperability settings; omitted by native Sandboxes." + x-kubernetes-validations: + - rule: "!has(self.sigsAgentSandbox) || self.sigsAgentSandbox != 'overlay' || has(self.upstreamSandboxRef)" + message: "overlay mode requires a same-namespace upstreamSandboxRef" + properties: + sigsAgentSandbox: + type: string + enum: ["off", "observe", "translate", "overlay"] + upstreamSandboxRef: + type: object + required: ["name"] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + aiConformanceReference: + type: boolean credentialsRef: type: object description: "Explicit agent credential collection in this Sandbox's workspace; replaces legacy credentials while set. Missing/replaced sources fail closed." diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md index 1c1b71a9..1c8ba3e3 100644 --- a/docs/security-audits/2026-09-07-credential-sources.md +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -71,6 +71,21 @@ legacy fallback; explicit opt-out restoring the preserved direct collection; and cleanup preserving the core namespace. Shell syntax is checked locally; execution of this new case awaits hosted CI. +Hosted Kind execution at `b69a6ad6` exposed a real installation blocker: +the new CEL rule referenced `upstreamCompatibility`, which was missing from the +handwritten Helm schema. Kubernetes rejected the entire Sandbox CRD, so no +credential-consumer lifecycle result was established. Local rendering and +source review were insufficient to catch that API-server compilation failure. + +The repair declares the existing Rust compatibility fields in the optional +Helm schema, retains the source/overlay admission guard, and requires an upstream +reference for overlay mode. The targeted schema assertion now checks the +referenced field definitions, not just the presence of rule text. The Kind +fixture also requires the intended overlay rejection message, and Helm setup +failure now stops the harness instead of producing cascading secondary failures. +Fifteen targeted schema/Helm tests, typecheck, scoped lint, Helm lint and shell +syntax pass locally; the repaired head still requires real API-server execution. + Diff-based publication gates still run on the parent's eventual atomic commit; the candidate was intentionally not committed or pushed here. No live Kubernetes admission or customer rollout was executed. Real-cluster diff --git a/tests/e2e/credential-sources.sh b/tests/e2e/credential-sources.sh index 58c5dd71..b9ae8058 100644 --- a/tests/e2e/credential-sources.sh +++ b/tests/e2e/credential-sources.sh @@ -29,8 +29,40 @@ wait_for_credential_consumer() { test_credential_sources() { local k=(kubectl --context kind-kars-e2e) - local source_uid projection_uid direct_uid namespace_uid projection + local source_uid projection_uid direct_uid namespace_uid projection admission namespace_uid=$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}') || return 1 + if admission=$("${k[@]}" create --dry-run=server -f - 2>&1 <<'YAML' +apiVersion: kars.azure.com/v1alpha1 +kind: KarsSandbox +metadata: + name: e2e-source-overlay + namespace: kars-system +spec: + inferenceRef: + name: e2e-source-inference + runtime: + kind: BYO + byo: + image: kars-sandbox-e2e:dev + contractVersion: v1 + sandbox: + isolation: standard + credentialsRef: + name: kars-credential-source-e2e-source-overlay + uid: fixture-uid + upstreamCompatibility: + sigsAgentSandbox: overlay + upstreamSandboxRef: + name: upstream-fixture +YAML + ); then + fail "Admission accepted source credentials on an overlay-managed runtime"; return 1 + fi + if ! printf '%s\n' "$admission" | grep -q "credentialsRef requires a controller-managed runtime"; then + printf '%s\n' "$admission" + fail "Overlay admission failed for a reason other than the intended source guard"; return 1 + fi + pass "The API server compiles the source schema and rejects overlay credentials for the intended reason" if ! "${k[@]}" create -f - <<'YAML' apiVersion: v1 kind: Secret diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 4be72545..9e9c58cc 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -188,6 +188,7 @@ install_crds() { kubectl get all -n kars-system || true kubectl describe pod -n kars-system -l app.kubernetes.io/component=controller || true kubectl logs -n kars-system -l app.kubernetes.io/component=controller --tail=200 || true + return 1 fi } From d444656bd74983cc7926d3fc2017ded106c76202 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 00:20:04 +0200 Subject: [PATCH 3/6] test(credentials): bound policy cleanup and retain timeout diagnostics The real consumer lifecycle passed but its final InferencePolicy deletion hung. Keep deletion mandatory, bound the wait and API calls, and surface safe diagnostics before the job deadline. Do not claim the underlying cleanup issue resolved. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/testing/credential-source-e2e.test.ts | 38 +++++++++++++++++++ .../2026-09-07-credential-sources.md | 10 +++++ tests/e2e/credential-sources.sh | 17 ++++++++- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 cli/src/testing/credential-source-e2e.test.ts diff --git a/cli/src/testing/credential-source-e2e.test.ts b/cli/src/testing/credential-source-e2e.test.ts new file mode 100644 index 00000000..3fc4cd57 --- /dev/null +++ b/cli/src/testing/credential-source-e2e.test.ts @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const harness = fileURLToPath(new URL("../../../tests/e2e/credential-sources.sh", import.meta.url)); + +describe("credential-source E2E cleanup bounds", () => { + it.each([0, 1])("propagates policy deletion exit %s without hiding failure behind diagnostics", exit => { + const result = spawnSync("bash", ["-c", ` + set -euo pipefail + source "$1" + fail() { printf 'FAIL: %s\\n' "$*" >&2; } + kubectl() { + printf 'CALL: %s\\n' "$*" >&2 + if [ "$3" = delete ]; then return "$POLICY_EXIT"; fi + return 1 + } + cleanup_credential_source_policy + `, "credential-cleanup", harness], { + encoding: "utf8", timeout: 5_000, + env: { ...process.env, POLICY_EXIT: String(exit) }, + }); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(exit); + const calls = result.stderr.split("\n").filter(line => line.startsWith("CALL:")); + expect(calls[0]).toBe( + "CALL: --context kind-kars-e2e delete inferencepolicy e2e-source-inference -n kars-system --timeout=90s --request-timeout=20s", + ); + expect(calls).toHaveLength(exit ? 4 : 1); + expect(calls.every(call => call.includes("--context kind-kars-e2e") && call.includes("--request-timeout=20s"))) + .toBe(true); + expect(result.stderr.includes("FAIL:")).toBe(exit !== 0); + expect(calls.some(call => call.includes("get secret"))).toBe(false); + }); +}); diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md index 1c8ba3e3..38afb2b2 100644 --- a/docs/security-audits/2026-09-07-credential-sources.md +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -86,6 +86,16 @@ failure now stops the harness instead of producing cascading secondary failures. Fifteen targeted schema/Helm tests, typecheck, scoped lint, Helm lint and shell syntax pass locally; the repaired head still requires real API-server execution. +The next hosted run at `693a46ca` installed the CRD and passed the intended +overlay admission rejection. The real BYO consumer then passed initial +delivery, rotation with key removal, source-deletion revocation without legacy +fallback, and explicit opt-out. Sandbox namespace cleanup also completed. +The run subsequently hit its job deadline while deleting the InferencePolicy +fixture; full lifecycle cleanup is therefore not qualified. The fixture now +bounds that deletion to 90 seconds and captures policy/ConfigMap/controller +diagnostics on failure rather than hanging until the job cancels. This is a +diagnostic change, not a claimed repair of the underlying cleanup failure. + Diff-based publication gates still run on the parent's eventual atomic commit; the candidate was intentionally not committed or pushed here. No live Kubernetes admission or customer rollout was executed. Real-cluster diff --git a/tests/e2e/credential-sources.sh b/tests/e2e/credential-sources.sh index b9ae8058..0618b015 100644 --- a/tests/e2e/credential-sources.sh +++ b/tests/e2e/credential-sources.sh @@ -4,6 +4,21 @@ # Uses only the existing disposable Kind cluster and its loaded BYO test image. # The consumer emits fixed markers, never environment values. +cleanup_credential_source_policy() { + local k=(kubectl --context kind-kars-e2e) + if ! "${k[@]}" delete inferencepolicy e2e-source-inference -n kars-system \ + --timeout=90s --request-timeout=20s >/dev/null; then + fail "Credential-source InferencePolicy cleanup did not complete within 90s" + "${k[@]}" get inferencepolicy e2e-source-inference -n kars-system \ + --request-timeout=20s -o yaml || true + "${k[@]}" get configmap inferencepolicy-e2e-source-inference-profile -n kars-system \ + --request-timeout=20s -o go-template='{{.metadata}}{{"\n"}}' || true + "${k[@]}" logs -n kars-system -l app.kubernetes.io/component=controller \ + --tail=1000 --since=5m --request-timeout=20s || true + return 1 + fi +} + wait_for_credential_consumer() { local marker="$1" deadline=$(($(date +%s) + 150)) pod namespace local k=(kubectl --context kind-kars-e2e) @@ -197,7 +212,7 @@ YAML "${k[@]}" delete karssandbox e2e-source -n kars-system --wait=false >/dev/null || return 1 "${k[@]}" wait --for=delete namespace/kars-e2e-source --timeout=120s || return 1 - "${k[@]}" delete inferencepolicy e2e-source-inference -n kars-system >/dev/null || return 1 + cleanup_credential_source_policy || return 1 if [ "$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}')" != "$namespace_uid" ]; then fail "Credential-source cleanup changed the core namespace"; return 1 fi From 8b206065608593667a40665b3f48225ef9ce278d Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 01:10:53 +0200 Subject: [PATCH 4/6] fix(controller): fence inference policy finalizer updates Replace partial server-side apply finalizer writes rejected by Kubernetes with named UID/resourceVersion-fenced metadata merges. Preserve unrelated finalizers and retain cleanup retries on non-404 profile deletion failures. Cover registration, deletion, conflicts and malformed identity through the real reconcile path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../src/inference_policy_finalizer_tests.rs | 204 ++++++++++++++++++ controller/src/inference_policy_reconciler.rs | 61 ++++-- .../2026-09-07-credential-sources.md | 12 ++ 3 files changed, 261 insertions(+), 16 deletions(-) create mode 100644 controller/src/inference_policy_finalizer_tests.rs diff --git a/controller/src/inference_policy_finalizer_tests.rs b/controller/src/inference_policy_finalizer_tests.rs new file mode 100644 index 00000000..d66d7835 --- /dev/null +++ b/controller/src/inference_policy_finalizer_tests.rs @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use serde_json::Value; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, +}; + +const POLICY_PATH: &str = + "/apis/kars.azure.com/v1alpha1/namespaces/workspace/inferencepolicies/example"; +const CONFIGMAP_PATH: &str = + "/api/v1/namespaces/workspace/configmaps/inferencepolicy-example-profile"; +const CUSTOMER_FINALIZER: &str = "customer.example/cleanup"; + +fn policy(deleting: bool) -> InferencePolicy { + let mut value = json!({ + "apiVersion": "kars.azure.com/v1alpha1", "kind": "InferencePolicy", + "metadata": { + "name": "example", "namespace": "workspace", "uid": "policy-uid", + "resourceVersion": "12", "finalizers": [CUSTOMER_FINALIZER], + }, + "spec": { "appliesTo": { "sandboxName": "agent" } }, + }); + if deleting { + value["metadata"]["deletionTimestamp"] = json!("2026-09-07T22:52:35Z"); + value["metadata"]["finalizers"] = json!([FINALIZER, CUSTOMER_FINALIZER]); + } + serde_json::from_value(value).unwrap() +} + +fn status(code: u16) -> ResponseTemplate { + ResponseTemplate::new(code).set_body_json(json!({ + "apiVersion": "v1", "kind": "Status", + "status": if code < 400 { "Success" } else { "Failure" }, + "reason": match code { + 404 => "NotFound", 409 => "Conflict", 403 => "Forbidden", _ => "InternalError" + }, + "code": code, + })) +} + +async fn setup() -> (MockServer, Arc) { + let server = MockServer::start().await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let client = Client::try_from(kube::Config::new(server.uri().parse().unwrap())).unwrap(); + let ctx = Arc::new(Ctx { + client: client.clone(), + http: reqwest::Client::new(), + phase_reporter: PhaseEventReporter::new(client, "InferencePolicy"), + }); + (server, ctx) +} + +async fn expect_patch(server: &MockServer, policy: &InferencePolicy, finalizers: Vec) { + let expected = json!({"metadata": { + "name": "example", "namespace": "workspace", "uid": "policy-uid", + "resourceVersion": "12", "finalizers": finalizers, + }}); + let mut response = policy.clone(); + response.metadata.resource_version = Some("13".into()); + response.metadata.finalizers = Some(finalizers); + Mock::given(method("PATCH")) + .and(path(POLICY_PATH)) + .and(wiremock::matchers::header( + "content-type", + "application/merge-patch+json", + )) + .and(wiremock::matchers::body_json(expected)) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .expect(1) + .mount(server) + .await; +} + +#[tokio::test] +async fn finalizer_registration_preserves_customer_metadata_without_partial_policy_apply() { + let policy = policy(false); + let (server, ctx) = setup().await; + expect_patch( + &server, + &policy, + vec![CUSTOMER_FINALIZER.into(), FINALIZER.into()], + ) + .await; + assert_eq!( + reconcile(Arc::new(policy), ctx).await.unwrap(), + Action::requeue(Duration::from_secs(1)) + ); + assert_eq!(server.received_requests().await.unwrap().len(), 1); +} + +#[tokio::test] +async fn deletion_removes_only_own_finalizer_after_profile_cleanup_or_notfound() { + for code in [200, 404] { + let policy = policy(true); + let (server, ctx) = setup().await; + Mock::given(method("DELETE")) + .and(path(CONFIGMAP_PATH)) + .respond_with(status(code)) + .expect(1) + .mount(&server) + .await; + expect_patch(&server, &policy, vec![CUSTOMER_FINALIZER.into()]).await; + assert_eq!( + reconcile(Arc::new(policy), ctx).await.unwrap(), + Action::await_change() + ); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method, "DELETE"); + assert_eq!(requests[1].method, "PATCH"); + assert!( + requests[1] + .url + .query_pairs() + .all(|(key, value)| key != "force" || value != "true") + ); + } +} + +#[tokio::test] +async fn profile_cleanup_errors_keep_finalizer_and_use_error_retry() { + for code in [403, 409, 500] { + let policy = Arc::new(policy(true)); + let (server, ctx) = setup().await; + Mock::given(method("DELETE")) + .and(path(CONFIGMAP_PATH)) + .respond_with(status(code)) + .expect(1) + .mount(&server) + .await; + let error = reconcile(policy.clone(), ctx.clone()).await.unwrap_err(); + assert!(matches!(&error, ReconcileError::Kube(kube::Error::Api(s)) if s.code == code)); + assert_ne!( + error_policy(policy.clone(), &error, ctx), + Action::await_change() + ); + assert!( + policy + .metadata + .finalizers + .as_ref() + .unwrap() + .contains(&FINALIZER.into()) + ); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } +} + +#[tokio::test] +async fn finalizer_update_conflicts_do_not_force_or_retry_stale_identity() { + for deleting in [false, true] { + let policy = Arc::new(policy(deleting)); + let (server, ctx) = setup().await; + if deleting { + Mock::given(method("DELETE")) + .and(path(CONFIGMAP_PATH)) + .respond_with(status(404)) + .expect(1) + .mount(&server) + .await; + } + Mock::given(method("PATCH")) + .and(path(POLICY_PATH)) + .respond_with(status(409)) + .expect(1) + .mount(&server) + .await; + let error = reconcile(policy.clone(), ctx.clone()).await.unwrap_err(); + assert_ne!(error_policy(policy, &error, ctx), Action::await_change()); + let requests = server.received_requests().await.unwrap(); + let patch = requests.last().unwrap(); + assert_eq!( + patch.headers.get("content-type").unwrap(), + "application/merge-patch+json" + ); + let body: Value = serde_json::from_slice(&patch.body).unwrap(); + assert_eq!(body["metadata"]["uid"], "policy-uid"); + assert_eq!(body["metadata"]["resourceVersion"], "12"); + assert!( + patch + .url + .query_pairs() + .all(|(key, value)| key != "force" || value != "true") + ); + assert_eq!(requests.len(), if deleting { 2 } else { 1 }); + } +} + +#[tokio::test] +async fn finalizer_patch_refuses_missing_api_identity_without_writing() { + for field in ["name", "namespace", "uid", "resourceVersion"] { + let (server, ctx) = setup().await; + let mut value = serde_json::to_value(policy(false)).unwrap(); + value["metadata"].as_object_mut().unwrap().remove(field); + let policy: InferencePolicy = serde_json::from_value(value).unwrap(); + let api = Api::namespaced(ctx.client.clone(), "workspace"); + let result = patch_finalizers(&api, &policy, vec![FINALIZER.into()]).await; + assert!(matches!(result, Err(ReconcileError::MissingIdentity(_)))); + assert!(server.received_requests().await.unwrap().is_empty()); + } +} diff --git a/controller/src/inference_policy_reconciler.rs b/controller/src/inference_policy_reconciler.rs index 5750d1ed..7cd4c67a 100644 --- a/controller/src/inference_policy_reconciler.rs +++ b/controller/src/inference_policy_reconciler.rs @@ -86,6 +86,8 @@ enum ReconcileError { Kube(#[from] kube::Error), #[error("JSON serialization error: {0}")] SerdeJson(#[from] serde_json::Error), + #[error("InferencePolicy metadata is missing {0}")] + MissingIdentity(&'static str), } impl ReconcileError { @@ -95,6 +97,7 @@ impl ReconcileError { match self { ReconcileError::Kube(_) => "kube_api", ReconcileError::SerdeJson(_) => "serde", + ReconcileError::MissingIdentity(_) => "identity", } } } @@ -124,13 +127,9 @@ async fn reconcile(policy: Arc, ctx: Arc) -> Result, + policy: &InferencePolicy, + finalizers: Vec, +) -> Result<(), ReconcileError> { + let required = |value: Option<&str>, field| { + value + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .ok_or(ReconcileError::MissingIdentity(field)) + }; + let name = required(policy.metadata.name.as_deref(), "name")?; + let namespace = required(policy.metadata.namespace.as_deref(), "namespace")?; + let uid = required(policy.metadata.uid.as_deref(), "UID")?; + let resource_version = required( + policy.metadata.resource_version.as_deref(), + "resourceVersion", + )?; + // Finalizers are metadata, not an apply-owned partial policy. Fence the + // update so a stale watch event cannot remove another writer's finalizers. + api.patch( + &name, + &PatchParams::default(), + &Patch::Merge(json!({"metadata": { + "name": name, "namespace": namespace, "uid": uid, + "resourceVersion": resource_version, "finalizers": finalizers + }})), + ) + .await?; + Ok(()) +} + async fn finalize( api: &Api, configmaps: &Api, @@ -773,7 +804,7 @@ async fn finalize( name: &str, ) -> Result { let cm_name = format!("inferencepolicy-{name}-profile"); - let _ = configmaps + configmaps .delete(&cm_name, &Default::default()) .await .map(|_| ()) @@ -783,7 +814,7 @@ async fn finalize( } else { Err(e) } - }); + })?; let finalizers: Vec = policy .metadata @@ -791,13 +822,7 @@ async fn finalize( .as_ref() .map(|v| v.iter().filter(|f| *f != FINALIZER).cloned().collect()) .unwrap_or_default(); - let patch = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"InferencePolicy","metadata":{"finalizers": finalizers}}); - api.patch( - name, - &PatchParams::apply(FIELD_MANAGER).force(), - &Patch::Apply(patch), - ) - .await?; + patch_finalizers(api, policy, finalizers).await?; tracing::info!(inferencepolicy = %name, "InferencePolicyDeleted"); Ok(Action::await_change()) } @@ -878,6 +903,10 @@ pub async fn run(client: Client) -> Result<()> { Ok(()) } +#[cfg(test)] +#[path = "inference_policy_finalizer_tests.rs"] +mod finalizer_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md index 38afb2b2..f5d4c710 100644 --- a/docs/security-audits/2026-09-07-credential-sources.md +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -96,6 +96,18 @@ bounds that deletion to 90 seconds and captures policy/ConfigMap/controller diagnostics on failure rather than hanging until the job cancels. This is a diagnostic change, not a claimed repair of the underlying cleanup failure. +The bounded follow-up at `d444656b` identified the cleanup cause: the +InferencePolicy reconciler sent a partial server-side-apply finalizer payload +without `metadata.name`, which the API server rejected with HTTP 400 after the +profile ConfigMap had already been deleted. The same error affected SRE's +inference policy. The repair uses complete name/namespace/UID/resourceVersion +metadata in merge patches for finalizer registration and removal, preserving +unrelated finalizers. Profile cleanup errors other than 404 now propagate before +the cleanup finalizer can be released. No force apply or finalizer bypass is +used. Fourteen focused reconciler tests, including HTTP request shape, +cleanup ordering and API/CAS error cases, and strict controller all-target +Clippy pass. Real Kind cleanup and independent repair review remain pending. + Diff-based publication gates still run on the parent's eventual atomic commit; the candidate was intentionally not committed or pushed here. No live Kubernetes admission or customer rollout was executed. Real-cluster From 3f04c656ba74492371edef5924290d1ff785d9f7 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 15:49:02 +0200 Subject: [PATCH 5/6] docs(security): record maintainer sign-off for PR 549 Record the explicit 2026-09-08 maintainer approval for the qualified source. Preserve the requirement for a separate independent human sign-off and all release boundaries. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../2026-09-07-credential-sources.md | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md index f5d4c710..d530f08b 100644 --- a/docs/security-audits/2026-09-07-credential-sources.md +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -1,9 +1,9 @@ # Agent credential-source capability review — 2026-09-07 -**Status:** additive candidate; independent security review and human sign-off -pending. Automated tests are evidence, not approval. No reviewer identity, -organizational approval, compliance certification, or production qualification -is asserted by this document. +**Status:** additive candidate with maintainer sign-off received; independent +human security review and sign-off remain pending. Automated tests are evidence, +not another person's approval. No organizational approval, compliance +certification, or production qualification is asserted by this document. ## Scope and base @@ -130,4 +130,21 @@ qualification and independent/human review remain release gates. 6. Review schema/controller mixed-version handling, consumer rollout behavior, and source-authoring permissions before release. -**Human sign-off: pending. Independent security sign-off: pending.** +## Qualification and maintainer sign-off update + +The earlier pending technical evidence above is superseded by closure at +`8b206065608593667a40665b3f48225ef9ce278d` in Azure/kars#549. Automated bounded +review closed the finalizer repair, and all hosted technical gates passed. +The real disposable Kind lifecycle completed 111 cases with zero failures, +including consumer rotation/revocation, explicit opt-out, mandatory policy +cleanup and preservation of the core namespace: +https://github.com/Azure/kars/actions/runs/34169168508/job/101889685225. +This is not a live customer deployment or Bridge credential/RBAC closure. + +The maintainer explicitly signed off and approved that qualified source on +2026-09-08. This does not approve later functional changes, other unresolved +slices, customer deployment or promotion to `main`. The separate independent +reviewer must provide their own sign-off before the two-person audit gate can +pass; this single maintainer approval must not be counted twice. + +Signed-off-by: pallakatos <191481949+pallakatos@users.noreply.github.com> From 135d8b4b303174c040fab0b3c4a753f7efb7e081 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 16:00:52 +0200 Subject: [PATCH 6/6] docs(security): record author integration waiver for PR 549 Record the author's explicit waiver of the second-person signature for this qualified integration slice only. Preserve truthful CI results, all other gates, exact source scope and immediate protection restoration. No main promotion or customer deployment is authorized. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../2026-09-07-credential-sources.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md index d530f08b..0241faae 100644 --- a/docs/security-audits/2026-09-07-credential-sources.md +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -148,3 +148,19 @@ reviewer must provide their own sign-off before the two-person audit gate can pass; this single maintainer approval must not be counted twice. Signed-off-by: pallakatos <191481949+pallakatos@users.noreply.github.com> + +## Explicit author waiver for integration assembly + +On 2026-09-08, Kars author `pallakatos` explicitly waived the second-person +sign-off for Azure/kars#549: "just push them and say I waived it". +This supersedes the independent-signature landing requirement above only for +assembling the already-qualified source into `Azure/kars:kars-bridge`. +It does not assert that an independent human review occurred. + +All other required technical and security gates must pass on the landing head. +The signature-only branch-protection exception and the already-authorized +account-specific review allowance must be restored immediately after the merge, +including on failure. No CI result is rewritten as successful. +The waiver does not cover functional changes beyond the qualified source, +later unresolved slices, `main` promotion, customer deployments or public +publication of the private Bridge application.