diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index 55c5a01e..77f0efe1 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -7,6 +7,8 @@ import ora from "ora"; import { loadContext, resolveSecret } from "../config.js"; import { assertRuntimeWired, buildRuntimeBlock, flagToKind } from "../runtime.js"; import { CLAIM, prepareCredentialNamespace } from "../lib/namespace-ownership.js"; +import { applySourceSandbox, prepareCredentialSource, updatesFromFlags, waitForCredentialSource, type SourceRef } from "../lib/credential-source.js"; +import { FLAG_ENV, SOURCE_KEYS, targetName } from "../lib/credential-source-io.js"; import { buildInferencePolicy, buildToolPolicy, @@ -20,6 +22,8 @@ export function addCommand(): Command { cmd .description("Add a new sandboxed agent to an existing kars cluster") .argument("", "Name for the new sandbox agent") + .option("--namespace ", "Workspace for the Sandbox and policy CRs", "kars-system") + .option("--credential-source", "Use a pinned workspace credential source instead of runtime-namespace credentials") // ── Core (all runtimes) ──────────────────────────────────────────── .option("--runtime ", "Runtime kind: openclaw | openai-agents | microsoft-agent-framework | langgraph | anthropic | pydantic-ai | hermes | byo", "openclaw") @@ -87,6 +91,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. `) .action(async (name: string, options) => { const { execa } = await import("execa"); + targetName(name, options.namespace); const runtimeKind = flagToKind(options.runtime); assertRuntimeWired(runtimeKind); @@ -110,7 +115,11 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. ["--image", options.image], ]; if (runtimeKind !== "OpenClaw") { - const used = openClawOnlyFlags.filter(([, v]) => v !== undefined && v !== "" && v !== false).map(([f]) => f); + const sourceFlags = options.credentialSource ? new Set(Object.entries(FLAG_ENV) + .filter(([, env]) => (SOURCE_KEYS as readonly string[]).includes(env)) + .map(([flag]) => `--${flag.replace(/[A-Z]/g, letter => `-${letter.toLowerCase()}`)}`)) : new Set(); + const used = openClawOnlyFlags.filter(([flag, value]) => !sourceFlags.has(flag) + && value !== undefined && value !== "" && value !== false).map(([flag]) => flag); if (used.length > 0) { console.error(chalk.red(`\n Error: ${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with --runtime openclaw.`)); console.error(chalk.dim(` Channels, skills, and plugin API keys are OpenClaw-specific entrypoint features.`)); @@ -146,7 +155,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. kind: "KarsSandbox", metadata: { name, - namespace: "kars-system", + namespace: options.namespace, }, spec: { runtime: runtimeBlock, @@ -292,7 +301,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. // S13: build companion same-namespace policy CRs (sibling to KarsSandbox). const inferencePolicy = buildInferencePolicy({ sandboxName: name, - namespace: "kars-system", + namespace: options.namespace, model: options.model, provider: "azure-ai-foundry", contentSafety: true, @@ -305,7 +314,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. const toolPolicy = options.governance ? buildToolPolicy({ sandboxName: name, - namespace: "kars-system", + namespace: options.namespace, profile: options.policyProfile || "default", }) : undefined; @@ -316,6 +325,11 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. const yaml = JSON.stringify(bundle, null, 2); if (options.dryRun) { + if (options.credentialSource) { + console.log(`Plan: create/update the opted-in source in ${options.namespace}, then bind its API-assigned UID when submitting ${name}.`); + console.log("No runnable source-bound manifest is emitted before a real UID exists. No credential values are printed."); + return; + } console.log(chalk.bold("\nKarsSandbox manifest (dry-run):\n")); console.log(yaml); console.log(chalk.dim("\nApply with: kubectl apply -f ")); @@ -447,10 +461,21 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. const allSecrets = { ...channelEnvSecrets, ...pluginSecrets, + ...(options.credentialSource ? updatesFromFlags(options) : {}), }; - if (Object.keys(allSecrets).length > 0) { + let sourceReference: SourceRef | undefined; + if (options.credentialSource) { + const prepared = await prepareCredentialSource(execa, name, options.namespace, allSecrets); + const policies = bundle.filter(item => item !== sandbox); + await execa("kubectl", ["apply", "-f", "-"], { + input: JSON.stringify({ apiVersion: "v1", kind: "List", items: policies }), + stdio: ["pipe", "pipe", "pipe"], + }); + await applySourceSandbox(execa, sandbox, prepared); + sourceReference = prepared.reference; + } else if (Object.keys(allSecrets).length > 0) { spinner.text = "Creating credential secret..."; - const namespaceUid = await prepareCredentialNamespace(execa, name, "kars-system"); + const namespaceUid = await prepareCredentialNamespace(execa, name, options.namespace); const metadata = sandbox.metadata as Record; metadata.annotations = { ...(metadata.annotations as Record | undefined), @@ -479,10 +504,14 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. kind: "List", items: bundle, }; - await execa("kubectl", ["apply", "-f", "-"], { - input: JSON.stringify(bundleManifest), - stdio: ["pipe", "pipe", "pipe"], - }); + if (sourceReference) { + await waitForCredentialSource(execa, name, options.namespace, sourceReference); + } else { + await execa("kubectl", ["apply", "-f", "-"], { + input: JSON.stringify(bundleManifest), + stdio: ["pipe", "pipe", "pipe"], + }); + } // The controller auto-mounts -credentials secret via envFrom (optional: true). // If the secret exists, env vars are injected into the sandbox container at startup. @@ -602,7 +631,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. if (options.skills) { console.log(chalk.dim(` Skills: ${options.skills}`)); } - console.log(chalk.dim(` Status: kubectl get karssandbox ${name} -n kars-system`)); + console.log(chalk.dim(` Status: kubectl get karssandbox ${name} -n ${options.namespace}`)); console.log(chalk.dim(` Connect: kars connect ${name}`)); console.log(chalk.dim(` Remove: kars destroy ${name}\n`)); diff --git a/cli/src/commands/credentials-source.test.ts b/cli/src/commands/credentials-source.test.ts new file mode 100644 index 00000000..fa203ed1 --- /dev/null +++ b/cli/src/commands/credentials-source.test.ts @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { credentialsCommand } from "./credentials.js"; +import { addCommand } from "./add.js"; + +const mocks = vi.hoisted(() => ({ + source: vi.fn(), direct: vi.fn(), execute: vi.fn(), + spinner: { start: vi.fn(), succeed: vi.fn(), fail: vi.fn(), warn: vi.fn() }, +})); +vi.mock("execa", () => ({ execa: mocks.execute })); +vi.mock("ora", () => ({ default: () => mocks.spinner })); +vi.mock("../config.js", async importOriginal => ({ + ...await importOriginal(), + resolveSecret: (_value: string | undefined) => undefined, + loadContext: () => undefined, +})); +vi.mock("../lib/credential-source.js", async importOriginal => ({ + ...await importOriginal(), + updateCredentialSource: mocks.source, updateDirectCredentials: mocks.direct, +})); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.spinner.start.mockReturnValue(mocks.spinner); + mocks.execute.mockResolvedValue({ stdout: "" }); + mocks.direct.mockResolvedValue(undefined); + mocks.source.mockResolvedValue({ kind: "source", reference: { name: "source", uid: "source-uid" } }); + vi.spyOn(console, "log").mockImplementation(() => {}); +}); +afterEach(() => vi.restoreAllMocks()); + +describe("credentials command source integration", () => { + it("routes opt-in flags to the workspace and never prints source values or restarts directly", async () => { + await credentialsCommand().parseAsync([ + "update", "demo", "--namespace", "workspace-a", "--use-source", "--telegram-token", "SENSITIVE-INPUT", + ], { from: "user" }); + expect(mocks.source).toHaveBeenCalledWith(mocks.execute, "demo", "workspace-a", { + updates: { TELEGRAM_BOT_TOKEN: "SENSITIVE-INPUT" }, remove: [], + useSource: true, disableSource: undefined, restart: true, + }); + expect(mocks.direct).not.toHaveBeenCalled(); + expect(mocks.execute).not.toHaveBeenCalled(); + expect(vi.mocked(console.log).mock.calls.flat().join(" ")).not.toContain("SENSITIVE-INPUT"); + }); + + it("supports remote removal and explicit source disable using existing credentials update", async () => { + await credentialsCommand().parseAsync(["update", "demo", "--remove", "telegram-token"], { from: "user" }); + expect(mocks.source.mock.calls[0][3].remove).toEqual(["TELEGRAM_BOT_TOKEN"]); + await credentialsCommand().parseAsync(["update", "demo", "--disable-source"], { from: "user" }); + expect(mocks.source.mock.calls[1][3].disableSource).toBe(true); + }); + + it("keeps legacy flags and --no-restart working on the direct path", async () => { + mocks.source.mockResolvedValue(undefined); + await credentialsCommand().parseAsync([ + "update", "demo", "--openai-api-key", "legacy-provider", "--no-restart", + ], { from: "user" }); + expect(mocks.direct).toHaveBeenCalledWith(mocks.execute, "demo", { OPENAI_API_KEY: "legacy-provider" }, []); + expect(mocks.execute).not.toHaveBeenCalled(); + }); +}); + +describe("source-mode add flags", () => { + it.each(["openclaw", "openai-agents", "microsoft-agent-framework", "langgraph", "anthropic", "pydantic-ai", "hermes"])( + "permits agent credential environment flags for %s without publishing a fake UID", async runtime => { + await addCommand().parseAsync([ + "demo", "--runtime", runtime, "--credential-source", "--telegram-token", "SENSITIVE-INPUT", "--dry-run", + ], { from: "user" }); + expect(mocks.execute).not.toHaveBeenCalled(); + const output = vi.mocked(console.log).mock.calls.flat().join(" "); + expect(output).toContain("No runnable source-bound manifest"); + expect(output).not.toContain("SENSITIVE-INPUT"); + }, + ); +}); diff --git a/cli/src/commands/credentials.ts b/cli/src/commands/credentials.ts index 4b689b29..3fe97775 100644 --- a/cli/src/commands/credentials.ts +++ b/cli/src/commands/credentials.ts @@ -3,6 +3,7 @@ import { Command } from "commander"; import chalk from "chalk"; +import { removedKeys, updateCredentialSource, updateDirectCredentials } from "../lib/credential-source.js"; import { banner, section } from "../stepper.js"; import { promptAndSaveCredentials, SECRETS_FILE, @@ -233,8 +234,12 @@ export function credentialsCommand(): Command { // Subcommand: update credentials for a running AKS sandbox const update = new Command("update"); update - .description("Update credentials for a running AKS sandbox (updates secret + restarts pod)") + .description("Update sandbox credentials (direct Secret or pinned workspace source)") .argument("", "Sandbox name") + .option("--namespace ", "Workspace containing the Sandbox CR", "kars-system") + .option("--use-source", "Opt in to a UID-pinned workspace source (migrates direct credentials once)") + .option("--disable-source", "Explicitly restore the unchanged legacy credential collection") + .option("--remove ", "Remove keys (credential flag names or environment keys, comma separated)") .option("--telegram-token ", "New Telegram bot token") .option("--telegram-allow-from ", "Telegram allowed user IDs (comma-separated)") .option("--slack-token ", "New Slack bot token") @@ -269,40 +274,29 @@ export function credentialsCommand(): Command { if (options[flag]) updates[env] = options[flag]; } - if (Object.keys(updates).length === 0) { + const remove = removedKeys(options.remove); + if (Object.keys(updates).length === 0 && remove.length === 0 && !options.useSource && !options.disableSource) { console.error(chalk.red(" No credentials specified. Use --telegram-token, --brave-api-key, etc.")); process.exit(1); } const namespace = `kars-${name}`; - const secretName = `${name}-credentials`; const spinner = ora(`Updating credentials for '${name}'...`).start(); try { - // Read existing secret (if any) and merge with new values - let existing: Record = {}; - try { - const { stdout } = await execa("kubectl", [ - "get", "secret", secretName, "-n", namespace, - "-o", "jsonpath={.data}", - ], { stdio: "pipe" }); - if (stdout && stdout !== "{}") { - const data = JSON.parse(stdout); - for (const [k, v] of Object.entries(data)) { - existing[k] = Buffer.from(v as string, "base64").toString(); - } - } - } catch { /* secret doesn't exist yet */ } - - const merged = { ...existing, ...updates }; - - // Create/replace the secret - const secretArgs = ["create", "secret", "generic", secretName, "-n", namespace, "--dry-run=client", "-o", "yaml"]; - for (const [env, val] of Object.entries(merged)) { - secretArgs.push(`--from-literal=${env}=${val}`); + const source = await updateCredentialSource(execa, name, options.namespace, { + updates, remove, useSource: options.useSource, disableSource: options.disableSource, + restart: options.restart, + }); + if (source) { + spinner.succeed(source.staged + ? `Source staged; bind it with kars add ${name} --credential-source` + : source.reference ? "Credential source reconciled; controller owns runtime refresh" + : "Source disabled; controller will restore direct credentials"); + if (source.reference) console.log(chalk.dim(` Source: ${options.namespace}/${source.reference.name} (UID ${source.reference.uid})`)); + return; } - const { stdout: yaml } = await execa("kubectl", secretArgs, { stdio: "pipe" }); - await execa("kubectl", ["apply", "-f", "-"], { input: yaml, stdio: ["pipe", "pipe", "pipe"] }); + await updateDirectCredentials(execa, name, updates, remove); spinner.succeed("Secret updated"); @@ -310,6 +304,7 @@ export function credentialsCommand(): Command { for (const [env, val] of Object.entries(updates)) { console.log(chalk.dim(` ${env} = ••••${val.slice(-4)}`)); } + for (const key of remove) console.log(chalk.dim(` Removed ${key}`)); // Restart pod unless --no-restart if (options.restart !== false) { diff --git a/cli/src/lib/credential-source-io.ts b/cli/src/lib/credential-source-io.ts new file mode 100644 index 00000000..af4b5c0d --- /dev/null +++ b/cli/src/lib/credential-source-io.ts @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Execute } from "./deployment-target.js"; + +export interface Metadata { + name: string; namespace?: string; uid: string; resourceVersion: string; + annotations?: Record; + ownerReferences?: Array<{ apiVersion: string; kind: string; name: string; uid: string; controller?: boolean; blockOwnerDeletion?: boolean }>; + deletionTimestamp?: string; +} +export interface SourceRef { name: string; uid: string } +export interface Sandbox { + metadata: Metadata; + spec: { credentialsRef?: SourceRef | null; [key: string]: unknown }; + status?: { conditions?: Array<{ type: string; status: string; reason: string; message: string }> }; +} +export interface Secret { + apiVersion?: string; kind?: string; type?: string; + metadata: Metadata; data?: Record; +} +export const SOURCE_KEYS = [ + "TELEGRAM_BOT_TOKEN", "TELEGRAM_ALLOW_FROM", "SLACK_BOT_TOKEN", "DISCORD_BOT_TOKEN", + "WHATSAPP_ENABLED", "BRAVE_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "FIRECRAWL_API_KEY", + "PERPLEXITY_API_KEY", +] as const; +export const SOURCE = { + prefix: "kars-credential-source-", + purpose: "kars.azure.com/credential-purpose", + target: "kars.azure.com/credential-target", + intent: "kars.azure.com/credential-binding-intent", + sandboxUid: "kars.azure.com/credential-sandbox-uid", + workspace: "kars.azure.com/credential-workspace", + namespaceUid: "kars.azure.com/credential-namespace-uid", +} as const; +export const FLAG_ENV: Record = { + telegramToken: "TELEGRAM_BOT_TOKEN", telegramAllowFrom: "TELEGRAM_ALLOW_FROM", + slackToken: "SLACK_BOT_TOKEN", discordToken: "DISCORD_BOT_TOKEN", + braveApiKey: "BRAVE_API_KEY", tavilyApiKey: "TAVILY_API_KEY", exaApiKey: "EXA_API_KEY", + firecrawlApiKey: "FIRECRAWL_API_KEY", perplexityApiKey: "PERPLEXITY_API_KEY", openaiApiKey: "OPENAI_API_KEY", +}; + +export function targetName(name: string, workspace: string): string { + for (const value of [name, workspace]) { + if (value.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(value)) { + throw new Error("Credential target/workspace must be Kubernetes DNS labels"); + } + } + return `${SOURCE.prefix}${name}`; +} + +export function identity(meta: Metadata): void { + if (!meta?.name || !meta.uid || !meta.resourceVersion || meta.deletionTimestamp) { + throw new Error("Credential API identity is incomplete or terminating"); + } +} + +export async function run( + execute: Execute, stage: string, args: string[], input?: object, +): Promise { + try { + const result = await execute("kubectl", args, { + stdio: "pipe", ...(input ? { input: JSON.stringify(input) } : {}), + }); + return String(result.stdout); + } catch (error) { + // execa/admission errors can contain the entire Secret request. Never + // attach the original error, stdout, stderr, or command input as a cause. + const code = (error as { exitCode?: unknown }).exitCode; + throw new Error(`${stage} failed (kubectl exit ${typeof code === "number" ? code : "unknown"})`); + } +} + +export function parse(text: string): T { + try { return JSON.parse(text) as T; } + catch { throw new Error("Credential API returned invalid JSON"); } +} + +export async function get( + execute: Execute, kind: string, name: string, namespace?: string, +): Promise { + const text = await run(execute, "Read credential resource", [ + "get", kind, name, ...(namespace ? ["-n", namespace] : []), "--ignore-not-found", "-o", "json", + ]); + if (!text.trim()) return undefined; + const value = parse(text); + identity(value.metadata); + if (value.metadata.name !== name || (namespace && value.metadata.namespace !== namespace)) { + throw new Error("Credential API returned a different resource identity"); + } + return value; +} + +export function decode(secret?: Secret): Record { + const values: Record = Object.create(null); + for (const [key, value] of Object.entries(secret?.data ?? {})) { + try { + if (typeof value !== "string") throw new Error(); + const buffer = Buffer.from(value, "base64"); + if (buffer.toString("base64") !== value) throw new Error(); + values[key] = new TextDecoder("utf-8", { fatal: true }).decode(buffer); + } catch { throw new Error("Credential Secret has invalid encoded environment data"); } + } + return values; +} + +export function validateValues(values: Record): void { + let size = 0; + for (const [key, value] of Object.entries(values)) { + if (!(SOURCE_KEYS as readonly string[]).includes(key) || value.includes("\0")) { + throw new Error("Source mode accepts only supported agent channel/search credentials, not provider/control-plane or arbitrary environment keys"); + } + size += Buffer.byteLength(value); + } + if (size > 131_072) throw new Error("Credential source exceeds 128 KiB"); +} + +export function validateSource(secret: Secret, name: string, workspace: string, sandbox?: Sandbox): void { + identity(secret.metadata); + const meta = secret.metadata; + const annotations = meta.annotations ?? {}; + if (meta.name !== targetName(name, workspace) || meta.namespace !== workspace || secret.type !== "Opaque" + || annotations[SOURCE.purpose] !== "agent-source-v1" || annotations[SOURCE.target] !== name + || annotations[SOURCE.workspace] !== workspace || annotations[SOURCE.intent] !== "explicit-reference-v1") { + throw new Error("Existing source has incompatible purpose, type, or target; no takeover"); + } + const refs = meta.ownerReferences ?? []; + if (refs.length && (!sandbox || refs.length !== 1 || refs[0].apiVersion !== "kars.azure.com/v1alpha1" + || refs[0].kind !== "KarsSandbox" || refs[0].name !== name || refs[0].uid !== sandbox.metadata.uid + || refs[0].controller !== true || refs[0].blockOwnerDeletion !== false)) { + throw new Error("Source belongs to a different Sandbox incarnation"); + } + if (annotations[SOURCE.sandboxUid] !== undefined && annotations[SOURCE.sandboxUid] !== sandbox?.metadata.uid) { + throw new Error("Source Sandbox UID binding differs; no automatic reuse"); + } + if (annotations[SOURCE.namespaceUid] !== undefined + && annotations[SOURCE.namespaceUid] !== sandbox?.metadata.annotations?.["kars.azure.com/namespace-uid"]) { + throw new Error("Source runtime namespace binding differs"); + } +} + +export function updatesFromFlags(options: Record): Record { + return Object.fromEntries(Object.entries(FLAG_ENV) + .filter(([flag]) => typeof options[flag] === "string" && options[flag] !== "") + .map(([flag, env]) => [env, options[flag] as string])); +} + +export function removedKeys(value?: string): string[] { + return (value ?? "").split(",").map(key => key.trim()).filter(Boolean).map(key => { + const flag = key.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase()); + const env = FLAG_ENV[flag] ?? key; + if (!/^[A-Z_][A-Z0-9_]*$/.test(env)) throw new Error("Credential removal requires a flag name or environment key"); + return env; + }); +} diff --git a/cli/src/lib/credential-source.test.ts b/cli/src/lib/credential-source.test.ts new file mode 100644 index 00000000..37840447 --- /dev/null +++ b/cli/src/lib/credential-source.test.ts @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync } from "node:fs"; +import { describe, expect, it, vi } from "vitest"; +import type { Execute } from "./deployment-target.js"; +import { + applySourceSandbox, prepareCredentialSource, updateCredentialSource, + updateDirectCredentials, verifyCredentialReference, waitForCredentialSource, +} from "./credential-source.js"; +import { + SOURCE, SOURCE_KEYS, decode, removedKeys, targetName, validateValues, + type Sandbox, type Secret, +} from "./credential-source-io.js"; +import { CLAIM } from "./namespace-ownership.js"; + +const encode = (values: Record) => Object.fromEntries( + Object.entries(values).map(([key, value]) => [key, Buffer.from(value).toString("base64")]), +); +const reference = { name: "kars-credential-source-demo", uid: "source-uid" }; +const opts = (extra = {}) => ({ updates: {}, remove: [], ...extra }); + +function sandbox(): Sandbox { + return { + metadata: { name: "demo", namespace: "kars-system", uid: "sandbox-uid", resourceVersion: "10", + annotations: { [CLAIM.namespaceUid]: "namespace-uid" } }, + spec: { runtime: { kind: "OpenClaw", openclaw: {} } }, + }; +} + +function source(): Secret { + return { + apiVersion: "v1", kind: "Secret", type: "Opaque", + metadata: { + name: reference.name, namespace: "kars-system", uid: reference.uid, resourceVersion: "20", + annotations: { + [SOURCE.purpose]: "agent-source-v1", [SOURCE.target]: "demo", + [SOURCE.workspace]: "kars-system", [SOURCE.intent]: "explicit-reference-v1", + "customer.example/note": "preserve", + }, + }, + data: encode({ TELEGRAM_BOT_TOKEN: "initial-source" }), + }; +} + +function cluster() { + const state = { + sandbox: sandbox() as Sandbox | undefined, + source: undefined as Secret | undefined, + namespace: { + metadata: { + name: "kars-demo", uid: "namespace-uid", resourceVersion: "30", + annotations: { + [CLAIM.version]: "v1", [CLAIM.name]: "demo", [CLAIM.namespace]: "kars-system", + [CLAIM.uid]: "sandbox-uid", + }, + }, + } as { metadata: { name: string; uid: string; resourceVersion: string; annotations: Record } } | undefined, + legacy: { + apiVersion: "v1", kind: "Secret", type: "Opaque", + metadata: { name: "demo-credentials", namespace: "kars-demo", uid: "legacy-uid", resourceVersion: "40", + annotations: { "customer.example/keep": "yes" } }, + data: encode({ TELEGRAM_BOT_TOKEN: "legacy-token", SLACK_BOT_TOKEN: "keep-slack" }), + } as Secret | undefined, + failRead: false, failWrite: false, pruneRef: false, acknowledge: true, revision: 100, + recreateSourceOnPatch: false, conflictCreate: false, + }; + const run = vi.fn(async (command: string, args: readonly string[], options?: { input?: string }) => { + expect(command).toBe("kubectl"); + expect(args.some(arg => arg.startsWith("--from-literal"))).toBe(false); + if (args[0] === "get") { + if (state.failRead) throw Object.assign(new Error("forbidden: private-token-value"), { exitCode: 1 }); + let object: unknown; + if (args[1] === "karssandbox") { + if (state.acknowledge && state.sandbox?.spec.credentialsRef && state.source) { + state.sandbox.status = { conditions: [{ + type: "CredentialsReady", status: "True", reason: "Projected", + message: JSON.stringify({ sourceUid: state.source.metadata.uid, sourceVersion: state.source.metadata.resourceVersion }), + }] }; + } + object = state.sandbox; + } else if (args[1] === "namespace") object = state.namespace; + else object = args[2] === reference.name ? state.source : state.legacy; + if (!object) return { stdout: "" }; + if (args.includes("jsonpath={.metadata}")) object = (object as Secret).metadata; + return { stdout: JSON.stringify(object) }; + } + if (state.failWrite) throw Object.assign(new Error(`admission echoed ${options?.input}`), { exitCode: 1 }); + const body = JSON.parse(options!.input!); + if (args[0] === "create") { + if (state.conflictCreate) throw Object.assign(new Error("AlreadyExists"), { exitCode: 1 }); + if (body.kind === "Secret") { + expect(state.source).toBeUndefined(); + body.metadata.uid = reference.uid; + body.metadata.resourceVersion = String(++state.revision); + state.source = body; + } else { + expect(body.kind).toBe("KarsSandbox"); + expect(state.sandbox).toBeUndefined(); + body.metadata.uid = "sandbox-created"; + body.metadata.resourceVersion = String(++state.revision); + if (state.pruneRef) delete body.spec.credentialsRef; + state.sandbox = body; + } + return { stdout: JSON.stringify(body) }; + } + expect(args[0]).toBe("patch"); + expect(args).toContain("--patch-file=/dev/stdin"); + const object = args[1] === "karssandbox" ? state.sandbox + : args[2] === reference.name ? state.source : state.legacy; + expect(object).toBeDefined(); + if (state.recreateSourceOnPatch && object === state.source) { + object!.metadata.uid = "replacement"; + object!.metadata.resourceVersion = "replacement-version"; + } + if (object!.metadata.uid !== body.metadata.uid || object!.metadata.resourceVersion !== body.metadata.resourceVersion) { + throw Object.assign(new Error("Conflict"), { exitCode: 1 }); + } + if (args[1] === "karssandbox") { + if (state.pruneRef) delete body.spec.credentialsRef; + if (body.spec.credentialsRef === null) delete state.sandbox!.spec.credentialsRef; + else state.sandbox!.spec = { ...state.sandbox!.spec, ...body.spec }; + } else { + const secret = object as Secret; + secret.data ??= {}; + for (const [key, value] of Object.entries(body.data)) { + if (value === null) delete secret.data[key]; + else secret.data[key] = value as string; + } + } + object!.metadata.resourceVersion = String(++state.revision); + return { stdout: JSON.stringify(object) }; + }); + return { state, run, execute: run as unknown as Execute }; +} + +describe("credential source contract", () => { + it("matches the core and existing handoff agent-key allowlist", () => { + const rust = readFileSync(new URL("../../../controller/src/credential_source.rs", import.meta.url), "utf8"); + const block = rust.slice(rust.indexOf("pub const AGENT_KEYS"), rust.indexOf("pub fn source_name")); + expect([...block.matchAll(/"([A-Z][A-Z0-9_]+)"/g)].map(match => match[1])).toEqual([...SOURCE_KEYS]); + expect(SOURCE_KEYS).not.toContain("OPENAI_API_KEY"); + }); + + it.each(["OPENAI_API_KEY", "AZURE_CLIENT_SECRET", "AGT_KEY", "NODE_OPTIONS", "PATH"])( + "rejects reserved/provider key %s without echoing values", key => { + expect(() => validateValues({ [key]: "private-value" })).toThrow("supported agent"); + try { validateValues({ [key]: "private-value" }); } + catch (error) { expect(String(error)).not.toContain("private-value"); } + }, + ); + + it("normalizes removal keys and rejects path-shaped targets", () => { + expect(removedKeys("telegram-token,SLACK_BOT_TOKEN")).toEqual(["TELEGRAM_BOT_TOKEN", "SLACK_BOT_TOKEN"]); + expect(() => targetName("../other", "kars-system")).toThrow("DNS"); + expect(() => validateValues({ TELEGRAM_BOT_TOKEN: "a\0b" })).toThrow("environment"); + }); +}); + +describe("real source creation and migration", () => { + it("creates the source before the CR, pins its returned UID, and never writes a runtime namespace", async () => { + const { state, run, execute } = cluster(); + state.sandbox = undefined; state.namespace = undefined; state.legacy = undefined; + const prepared = await prepareCredentialSource(execute, "demo", "kars-system", { TELEGRAM_BOT_TOKEN: "sensitive-token-123" }); + expect(prepared.reference).toEqual(reference); + expect(state.sandbox).toBeUndefined(); + await applySourceSandbox(execute, { + apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", + metadata: { name: "demo", namespace: "kars-system" }, spec: { runtime: { kind: "OpenClaw", openclaw: {} } }, + }, prepared); + expect(state.sandbox).toMatchObject({ spec: { credentialsRef: reference } }); + const creates = run.mock.calls.filter(([, args]) => args[0] === "create"); + expect(creates).toHaveLength(2); + expect(JSON.parse(creates[0][2]!.input!).kind).toBe("Secret"); + expect(JSON.parse(creates[1][2]!.input!).kind).toBe("KarsSandbox"); + expect(run.mock.calls.some(([, args]) => args.includes("kars-demo"))).toBe(false); + expect(run.mock.calls.some(([, args]) => args.join(" ").includes("sensitive-token-123"))).toBe(false); + }); + + it("migrates the complete legacy collection once without changing its object or UID", async () => { + const { state, execute, run } = cluster(); + const legacy = structuredClone(state.legacy); + const result = await updateCredentialSource(execute, "demo", "kars-system", opts({ + useSource: true, updates: { TELEGRAM_BOT_TOKEN: "rotated" }, + })); + expect(result?.reference).toEqual(reference); + expect(decode(state.source)).toEqual({ TELEGRAM_BOT_TOKEN: "rotated", SLACK_BOT_TOKEN: "keep-slack" }); + expect(state.legacy).toEqual(legacy); + expect(state.sandbox?.spec.credentialsRef).toEqual(reference); + expect(run.mock.calls.filter(([, args]) => args.includes("kars-demo")).every(([, args]) => args[0] === "get")).toBe(true); + }); + + it("refuses unsafe/incomplete legacy migration before any source write", async () => { + const { state, execute, run } = cluster(); + state.legacy!.data!.OPENAI_API_KEY = Buffer.from("provider-value").toString("base64"); + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ useSource: true }))).rejects.toThrow("supported agent"); + expect(run.mock.calls.every(([, args]) => args[0] === "get")).toBe(true); + expect(state.source).toBeUndefined(); + }); + + it("does not silently attach a source after a racing CR create409 or schema pruning", async () => { + for (const prune of [false, true]) { + const { state, execute } = cluster(); + state.sandbox = undefined; state.namespace = undefined; + const prepared = await prepareCredentialSource(execute, "demo", "kars-system", {}); + state.pruneRef = prune; state.conflictCreate = !prune; + await expect(applySourceSandbox(execute, { + apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", + metadata: { name: "demo", namespace: "kars-system" }, spec: {}, + }, prepared)).rejects.toThrow(); + } + }); +}); + +describe("update, rotation, removal, and path selection", () => { + it("retains the direct path when no reference exists", async () => { + const { state, execute } = cluster(); + expect(await updateCredentialSource(execute, "demo", "kars-system", opts())).toBeUndefined(); + const uid = state.legacy!.metadata.uid; + await updateDirectCredentials(execute, "demo", { OPENAI_API_KEY: "legacy-supported" }, ["TELEGRAM_BOT_TOKEN"]); + expect(state.legacy!.metadata.uid).toBe(uid); + expect(decode(state.legacy)).toEqual({ SLACK_BOT_TOKEN: "keep-slack", OPENAI_API_KEY: "legacy-supported" }); + }); + + it("does not treat a wrong workspace or same-name conflict as the legacy direct path", async () => { + const { state, execute, run } = cluster(); + state.namespace!.metadata.annotations[CLAIM.namespace] = "other-workspace"; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow(); + state.sandbox = undefined; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow("--namespace"); + expect(run.mock.calls.every(([, args]) => args[0] === "get")).toBe(true); + }); + + it("updates/removes owned source keys without delete/recreate or legacy fallback", async () => { + const { state, execute, run } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + const legacy = structuredClone(state.legacy); + await updateCredentialSource(execute, "demo", "kars-system", opts({ + updates: { SLACK_BOT_TOKEN: "new-slack" }, remove: ["TELEGRAM_BOT_TOKEN"], + })); + expect(decode(state.source)).toEqual({ SLACK_BOT_TOKEN: "new-slack" }); + expect(state.source!.metadata.uid).toBe(reference.uid); + expect(state.source!.metadata.annotations!["customer.example/note"]).toBe("preserve"); + expect(state.legacy).toEqual(legacy); + expect(run.mock.calls.some(([, args]) => args.includes("kars-demo") || args[0] === "delete")).toBe(false); + }); + + it("requires explicit --use-source when the pinned source incarnation was replaced", async () => { + const { state, execute } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.source.metadata.uid = "new-source"; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow("missing/replaced"); + await updateCredentialSource(execute, "demo", "kars-system", opts({ useSource: true })); + expect(state.sandbox!.spec.credentialsRef?.uid).toBe("new-source"); + }); + + it("disables the reference without deleting source/customer data and preserves no-restart on direct mode", async () => { + const { state, execute } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + const original = structuredClone(state.source); + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ restart: false }))).rejects.toThrow("--no-restart"); + await updateCredentialSource(execute, "demo", "kars-system", opts({ disableSource: true })); + expect(state.sandbox!.spec.credentialsRef).toBeUndefined(); + expect(state.source).toEqual(original); + expect(await updateCredentialSource(execute, "demo", "kars-system", opts({ restart: false }))).toBeUndefined(); + }); + + it("supports explicit pre-CR staging without automatic delivery", async () => { + const { state, execute } = cluster(); + state.sandbox = undefined; state.namespace = undefined; + const result = await updateCredentialSource(execute, "demo", "kars-system", opts({ + useSource: true, updates: { TELEGRAM_BOT_TOKEN: "staged" }, + })); + expect(result?.staged).toBe(true); + expect(state.sandbox).toBeUndefined(); + expect(state.namespace).toBeUndefined(); + }); +}); + +describe("credential authority and truthful errors", () => { + it.each(["purpose", "owner", "namespace"])("rejects conflicting %s without takeover", async variant => { + const { state, execute, run } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + if (variant === "purpose") state.source.metadata.annotations![SOURCE.purpose] = "other"; + if (variant === "owner") state.source.metadata.annotations![SOURCE.sandboxUid] = "old-sandbox"; + if (variant === "namespace") state.source.metadata.namespace = "other-workspace"; + const original = structuredClone(state.source); + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ useSource: true }))).rejects.toThrow(); + expect(state.source).toEqual(original); + expect(run.mock.calls.every(([, args]) => args[0] === "get")).toBe(true); + }); + + it("propagates non404 errors and rejects UID/RV races without exposing request bodies", async () => { + const { state, execute } = cluster(); + state.failRead = true; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts())).rejects.toThrow("Read credential resource failed"); + state.failRead = false; state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.failWrite = true; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ + updates: { TELEGRAM_BOT_TOKEN: "private-token-value" }, + }))).rejects.not.toThrow("private-token-value"); + state.failWrite = false; state.recreateSourceOnPatch = true; + await expect(updateCredentialSource(execute, "demo", "kars-system", opts({ + updates: { TELEGRAM_BOT_TOKEN: "new-data" }, + }))).rejects.toThrow("Update credential source failed"); + expect(decode(state.source).TELEGRAM_BOT_TOKEN).toBe("initial-source"); + }); + + it("does not report activation when the controller has not acknowledged the source version", async () => { + const { state, execute } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.acknowledge = false; + await expect(waitForCredentialSource(execute, "demo", "kars-system", reference, { attempts: 1, delayMs: 0 })) + .rejects.toThrow("did not confirm"); + state.sandbox!.spec.credentialsRef = null; + await expect(verifyCredentialReference(execute, "demo", "kars-system", reference)).rejects.toThrow("did not retain"); + }); + + it("allows a repaired source to replace an older failure condition", async () => { + const { state, execute, run } = cluster(); + state.source = source(); state.sandbox!.spec.credentialsRef = reference; + state.acknowledge = false; + state.sandbox!.status = { conditions: [{ + type: "Degraded", status: "True", reason: "CredentialSourceUnavailable", message: "old failure", + }] }; + const original = run.getMockImplementation()!; + run.mockImplementation(async (...args) => { + const result = await original(...args); + if (args[1].includes("jsonpath={.metadata}")) state.acknowledge = true; + return result; + }); + await expect(waitForCredentialSource(execute, "demo", "kars-system", reference, { attempts: 2, delayMs: 0 })) + .resolves.toBeUndefined(); + }); +}); diff --git a/cli/src/lib/credential-source.ts b/cli/src/lib/credential-source.ts new file mode 100644 index 00000000..f7d3e3b8 --- /dev/null +++ b/cli/src/lib/credential-source.ts @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Execute } from "./deployment-target.js"; +import { CLAIM, namespaceClaimed, type OwnershipObject } from "./namespace-ownership.js"; +import { + SOURCE, decode, get, identity, parse, run, targetName, validateSource, validateValues, + type Metadata, type Sandbox, type Secret, type SourceRef, +} from "./credential-source-io.js"; + +export { FLAG_ENV, SOURCE_KEYS, removedKeys, updatesFromFlags } from "./credential-source-io.js"; +export type { SourceRef } from "./credential-source-io.js"; + +export interface PreparedSource { + reference: SourceRef; + sandbox?: Sandbox; +} + +function mergeValues(existing: Record, updates: Record, remove: string[]): Record { + const merged = { ...existing, ...updates }; + for (const key of remove) delete merged[key]; + return merged; +} + +async function legacyValues(execute: Execute, sandbox: Sandbox): Promise> { + const name = sandbox.metadata.name; + const target = `kars-${name}`; + const namespace = await get(execute, "namespace", target); + if (!namespace) { + if (sandbox.metadata.annotations?.["kars.azure.com/namespace-uid"]) { + throw new Error("Bound runtime namespace is missing; refusing an incomplete credential migration"); + } + return {}; + } + if (!namespaceClaimed(namespace, sandbox as unknown as OwnershipObject)) { + throw new Error("Credential migration requires namespace claim v1; finish namespace upgrade/adoption first"); + } + return decode(await get(execute, "secret", `${name}-credentials`, target)); +} + +async function writeSource( + execute: Execute, name: string, workspace: string, values: Record, existing?: Secret, sandbox?: Sandbox, +): Promise { + validateValues(values); + const sourceName = targetName(name, workspace); + const data: Record = Object.create(null); + for (const [key, value] of Object.entries(values)) data[key] = Buffer.from(value).toString("base64"); + let text: string; + if (existing) { + for (const key of Object.keys(existing.data ?? {})) if (!(key in values)) data[key] = null; + const { uid, resourceVersion } = existing.metadata; + text = await run(execute, "Update credential source", ["patch", "secret", sourceName, + "-n", workspace, "--type=merge", "--patch-file=/dev/stdin", "-o", "json"], { + metadata: { uid, resourceVersion }, data, + }); + } else { + text = await run(execute, "Create credential source", ["create", "-f", "-", "-o", "json"], { + apiVersion: "v1", kind: "Secret", type: "Opaque", + metadata: { + name: sourceName, namespace: workspace, + annotations: { + [SOURCE.purpose]: "agent-source-v1", [SOURCE.target]: name, + [SOURCE.workspace]: workspace, [SOURCE.intent]: "explicit-reference-v1", + }, + }, data, + }); + } + const result = parse(text); + identity(result.metadata); + if (result.metadata.name !== sourceName || result.metadata.namespace !== workspace + || (existing && result.metadata.uid !== existing.metadata.uid)) { + throw new Error("Credential source identity changed during write"); + } + validateSource(result, name, workspace, sandbox); + validateValues(decode(result)); + return { name: sourceName, uid: result.metadata.uid }; +} + +export async function prepareCredentialSource( + execute: Execute, name: string, workspace: string, updates: Record, + remove: string[] = [], useCurrentIncarnation = true, +): Promise { + const sourceName = targetName(name, workspace); + const sandbox = await get(execute, "karssandbox", name, workspace); + const reference = sandbox?.spec.credentialsRef; + if (reference && (reference.name !== sourceName || !reference.uid)) { + throw new Error("Sandbox credential reference is invalid; explicitly disable it before repair"); + } + const source = await get(execute, "secret", sourceName, workspace); + if (source) validateSource(source, name, workspace, sandbox); + if (reference && !useCurrentIncarnation && source?.metadata.uid !== reference.uid) { + throw new Error("Pinned credential source is missing/replaced; --use-source explicitly selects a new compatible incarnation"); + } + // Source mode is an entire collection. Migration is one-time and read-only + // against the existing target; active sources never fall back to legacy data. + const legacy = sandbox && !reference ? await legacyValues(execute, sandbox) : {}; + const values = mergeValues({ ...legacy, ...decode(source) }, updates, remove); + const selected = await writeSource(execute, name, workspace, values, source, sandbox); + return { reference: selected, sandbox }; +} + +export async function verifyCredentialReference( + execute: Execute, name: string, workspace: string, reference: SourceRef, +): Promise { + const sandbox = await get(execute, "karssandbox", name, workspace); + if (!sandbox || sandbox.spec.credentialsRef?.name !== reference.name + || sandbox.spec.credentialsRef.uid !== reference.uid) { + throw new Error("Cluster did not retain the pinned credential reference; upgrade the CRD/controller before using source mode"); + } + return sandbox; +} + +export async function applySourceSandbox( + execute: Execute, manifest: Record, prepared: PreparedSource, +): Promise { + const metadata = manifest.metadata as { name: string; namespace: string }; + const spec = { ...(manifest.spec as object), credentialsRef: prepared.reference }; + let written: Sandbox; + if (prepared.sandbox) { + const { uid, resourceVersion } = prepared.sandbox.metadata; + written = parse(await run(execute, "Bind existing Sandbox credentials", [ + "patch", "karssandbox", metadata.name, "-n", metadata.namespace, + "--type=merge", "--patch-file=/dev/stdin", "-o", "json", + ], { metadata: { uid, resourceVersion }, spec })); + if (written.metadata.uid !== uid) throw new Error("Sandbox identity changed while binding credentials"); + } else { + // CREATE, not apply: a racing Sandbox of the same name is not ours to bind. + written = parse(await run(execute, "Create source-bound Sandbox", ["create", "-f", "-", "-o", "json"], { + ...manifest, spec, + })); + } + identity(written.metadata); + const confirmed = await verifyCredentialReference(execute, metadata.name, metadata.namespace, prepared.reference); + if (confirmed.metadata.uid !== written.metadata.uid) throw new Error("Sandbox was recreated after source binding"); +} + +export async function waitForCredentialSource( + execute: Execute, name: string, workspace: string, reference: SourceRef, + options: { attempts?: number; delayMs?: number } = {}, +): Promise { + const attempts = options.attempts ?? 30; + for (let attempt = 0; attempt < attempts; attempt++) { + const sandbox = await verifyCredentialReference(execute, name, workspace, reference); + const text = await run(execute, "Read source reconciliation version", [ + "get", "secret", reference.name, "-n", workspace, "-o", "jsonpath={.metadata}", + ]); + const metadata = parse(text); + identity(metadata); + if (metadata.uid !== reference.uid) throw new Error("Credential source was replaced while awaiting reconciliation"); + const conditions = sandbox.status?.conditions ?? []; + const ready = conditions.find(condition => condition.type === "CredentialsReady" && condition.status === "True"); + if (ready) { + const observed = parse<{ sourceUid?: string; sourceVersion?: string }>(ready.message); + if (observed.sourceUid === reference.uid && observed.sourceVersion === metadata.resourceVersion) return; + } + // An earlier failure condition can precede this source revision. Allow the + // controller to process a repair rather than treating stale status as final. + if (attempt + 1 < attempts) await new Promise(resolve => setTimeout(resolve, options.delayMs ?? 3000)); + } + throw new Error("Controller did not confirm credential source reconciliation; check compatible controller/CRD versions and Sandbox conditions"); +} + +export async function updateCredentialSource( + execute: Execute, name: string, workspace: string, + options: { updates: Record; remove: string[]; useSource?: boolean; disableSource?: boolean; restart?: boolean }, +): Promise<{ kind: "source"; reference?: SourceRef; staged?: boolean } | undefined> { + targetName(name, workspace); + const sandbox = await get(execute, "karssandbox", name, workspace); + if (!sandbox?.spec.credentialsRef && !options.useSource && !options.disableSource) { + const namespace = await get(execute, "namespace", `kars-${name}`); + if (namespace?.metadata.annotations?.[CLAIM.version] !== undefined) { + if (!sandbox || !namespaceClaimed(namespace, sandbox as unknown as OwnershipObject)) { + throw new Error("Runtime namespace belongs to another or missing Sandbox; select its workspace with --namespace"); + } + } + return undefined; + } + if (options.restart === false) throw new Error("Source mode always refreshes runtime credentials; --no-restart is only supported for direct credentials"); + if (options.disableSource) { + if (!sandbox) throw new Error("Sandbox does not exist"); + if (options.useSource || options.remove.length || Object.keys(options.updates).length) { + throw new Error("--disable-source cannot be combined with credential changes"); + } + if (sandbox.spec.credentialsRef) { + const { uid, resourceVersion } = sandbox.metadata; + const result = parse(await run(execute, "Disable credential source", [ + "patch", "karssandbox", name, "-n", workspace, "--type=merge", "--patch-file=/dev/stdin", "-o", "json", + ], { metadata: { uid, resourceVersion }, spec: { credentialsRef: null } })); + if (result.spec.credentialsRef) throw new Error("Controller API did not remove the source reference"); + } + return { kind: "source" }; + } + const prepared = await prepareCredentialSource(execute, name, workspace, options.updates, options.remove, options.useSource === true); + if (prepared.sandbox && (prepared.sandbox.spec.credentialsRef?.uid !== prepared.reference.uid + || prepared.sandbox.spec.credentialsRef?.name !== prepared.reference.name)) { + await applySourceSandbox(execute, { + apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", + metadata: { name, namespace: workspace }, spec: prepared.sandbox.spec, + }, prepared); + } + if (prepared.sandbox) await waitForCredentialSource(execute, name, workspace, prepared.reference); + return { kind: "source", reference: prepared.reference, staged: !prepared.sandbox }; +} + +/** Preserve unrelated direct keys; explicit nulls remove keys across managers + * without deleting/recreating customer Secrets. */ +export async function updateDirectCredentials( + execute: Execute, name: string, updates: Record, remove: string[], +): Promise { + targetName(name, "kars-system"); + const namespace = `kars-${name}`; + const secretName = `${name}-credentials`; + const existing = await get(execute, "secret", secretName, namespace); + const data: Record = Object.create(null); + for (const [key, value] of Object.entries(updates)) data[key] = Buffer.from(value).toString("base64"); + for (const key of remove) data[key] = null; + if (existing) { + await run(execute, "Update direct credentials", [ + "patch", "secret", secretName, "-n", namespace, "--type=merge", "--patch-file=/dev/stdin", + ], { metadata: { uid: existing.metadata.uid, resourceVersion: existing.metadata.resourceVersion }, data }); + } else if (Object.keys(updates).some(key => !remove.includes(key))) { + for (const key of remove) delete data[key]; + await run(execute, "Create direct credentials", ["create", "-f", "-"], { + apiVersion: "v1", kind: "Secret", type: "Opaque", metadata: { name: secretName, namespace }, data, + }); + } +} diff --git a/cli/src/testing/credential-source-e2e.test.ts b/cli/src/testing/credential-source-e2e.test.ts new file mode 100644 index 00000000..3fc4cd57 --- /dev/null +++ b/cli/src/testing/credential-source-e2e.test.ts @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const harness = fileURLToPath(new URL("../../../tests/e2e/credential-sources.sh", import.meta.url)); + +describe("credential-source E2E cleanup bounds", () => { + it.each([0, 1])("propagates policy deletion exit %s without hiding failure behind diagnostics", exit => { + const result = spawnSync("bash", ["-c", ` + set -euo pipefail + source "$1" + fail() { printf 'FAIL: %s\\n' "$*" >&2; } + kubectl() { + printf 'CALL: %s\\n' "$*" >&2 + if [ "$3" = delete ]; then return "$POLICY_EXIT"; fi + return 1 + } + cleanup_credential_source_policy + `, "credential-cleanup", harness], { + encoding: "utf8", timeout: 5_000, + env: { ...process.env, POLICY_EXIT: String(exit) }, + }); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(exit); + const calls = result.stderr.split("\n").filter(line => line.startsWith("CALL:")); + expect(calls[0]).toBe( + "CALL: --context kind-kars-e2e delete inferencepolicy e2e-source-inference -n kars-system --timeout=90s --request-timeout=20s", + ); + expect(calls).toHaveLength(exit ? 4 : 1); + expect(calls.every(call => call.includes("--context kind-kars-e2e") && call.includes("--request-timeout=20s"))) + .toBe(true); + expect(result.stderr.includes("FAIL:")).toBe(exit !== 0); + expect(calls.some(call => call.includes("get secret"))).toBe(false); + }); +}); diff --git a/cli/src/testing/credential-source.test.ts b/cli/src/testing/credential-source.test.ts new file mode 100644 index 00000000..5045db9c --- /dev/null +++ b/cli/src/testing/credential-source.test.ts @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { parseAllDocuments } from "yaml"; + +const root = new URL("../../../", import.meta.url); +const source = (path: string) => readFileSync(new URL(path, root), "utf8"); + +describe("credential source public integration", () => { + it("renders an optional UID-pinned schema with same-target and managed-runtime admission guards", () => { + const yaml = execFileSync("helm", [ + "template", "kars", fileURLToPath(new URL("deploy/helm/kars", root)), + "--show-only", "templates/crd.yaml", + ], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 }); + const resource = parseAllDocuments(yaml).map(doc => { + if (doc.errors.length) throw doc.errors[0]; + return doc.toJSON(); + }).find(doc => doc?.metadata?.name === "karssandboxes.kars.azure.com"); + const schema = resource.spec.versions[0].schema.openAPIV3Schema; + const spec = schema.properties.spec; + expect(spec.required).not.toContain("credentialsRef"); + expect(spec.required).not.toContain("upstreamCompatibility"); + expect(spec.properties.credentialsRef.required).toEqual(["name", "uid"]); + // CEL compiles against declared schema fields, including those inside has(). + expect(spec.properties.upstreamCompatibility).toMatchObject({ + type: "object", + properties: { + sigsAgentSandbox: { type: "string", enum: ["off", "observe", "translate", "overlay"] }, + upstreamSandboxRef: { + type: "object", required: ["name"], + properties: { name: { type: "string", minLength: 1, maxLength: 253 } }, + }, + aiConformanceReference: { type: "boolean" }, + }, + }); + expect(spec.properties.upstreamCompatibility["x-kubernetes-validations"]) + .toContainEqual(expect.objectContaining({ + rule: "!has(self.sigsAgentSandbox) || self.sigsAgentSandbox != 'overlay' || has(self.upstreamSandboxRef)", + })); + expect(schema["x-kubernetes-validations"].some((rule: { rule: string }) => + rule.rule.includes("self.metadata.name") && rule.rule.includes("kars-credential-source-"))).toBe(true); + expect(spec["x-kubernetes-validations"].some((rule: { rule: string }) => + rule.rule.includes("credentialsRef") && rule.rule.includes("overlay"))).toBe(true); + }); + + it("mounts the projection in the common agent container, never the inference router", () => { + const code = source("controller/src/reconciler/mod.rs"); + const agent = code.slice(code.indexOf("let mut agent_container ="), code.indexOf("let mut agent_container =") + 1200); + expect(agent).toContain('"envFrom": credentials.env_from(&name)'); + expect(code.match(/credentials\.env_from/g)).toHaveLength(1); + expect(code).toContain('"envFrom": inference::provider_env_from()'); + expect(code.indexOf("credential_sources::reconcile(")).toBeLessThan(code.indexOf("let runtime_spec =")); + expect(code).toContain(".owns("); + expect(code.split("\n").length).toBeLessThan(3530); + }); + + it("keeps source values off argv and binds through a real UID rather than a generated placeholder", () => { + const helpers = source("cli/src/lib/credential-source.ts"); + expect(helpers).toContain("metadata: { uid, resourceVersion }"); + expect(helpers).not.toContain("--from-literal"); + expect(helpers).not.toContain("--force"); + const add = source("cli/src/commands/add.ts"); + expect(add.indexOf("await prepareCredentialSource")).toBeLessThan(add.indexOf("await applySourceSandbox")); + expect(add).toContain("await waitForCredentialSource"); + const projection = source("controller/src/reconciler/credential_source_projection.rs"); + expect(projection).not.toContain(".force()"); + expect(projection).toContain("Preconditions"); + }); + + it("keeps new production modules bounded and free of stubs, crypto primitives, or gate waivers", () => { + for (const path of [ + "controller/src/credential_source.rs", + "controller/src/reconciler/agent_env.rs", + "controller/src/reconciler/credential_sources.rs", + "controller/src/reconciler/credential_source_projection.rs", + "controller/src/reconciler/credential_source_workloads.rs", + "cli/src/lib/credential-source.ts", "cli/src/lib/credential-source-io.ts", + ]) { + const text = source(path); + expect(text.split("\n").length - 1, path).toBeLessThanOrEqual(800); + expect(text.slice(0, 150), path).toContain("Copyright (c) Microsoft Corporation"); + expect(text, path).not.toMatch(/TODO\b|FIXME\b|XXX\b|HACK\b|unimplemented!\(|\btodo!\(|\bplaceholder\b|ci:stub-ok|ci:loc-ok/); + expect(text, path).not.toMatch(/^use (sha2|hmac|curve25519_dalek|ed25519_dalek|x25519_dalek|aes|chacha20poly1305)::/m); + expect(text, path).not.toMatch(/crypto\.subtle\.sign|createHmac|createSign|@noble\/(curves|hashes)/); + } + }); +}); diff --git a/controller/src/crd.rs b/controller/src/crd.rs index 8eb458c1..36c89d02 100644 --- a/controller/src/crd.rs +++ b/controller/src/crd.rs @@ -72,6 +72,11 @@ pub struct KarsSandboxSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_ref: Option, + /// Explicit agent credential source in this CR's workspace. Source mode + /// replaces the legacy credential collection; no cross-namespace refs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credentials_ref: Option, + /// Network policy pub network_policy: Option, diff --git a/controller/src/credential_source.rs b/controller/src/credential_source.rs new file mode 100644 index 00000000..d6267529 --- /dev/null +++ b/controller/src/credential_source.rs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Public, opt-in agent credential-source contract. This is not a generic +//! Secret reference: the reserved name, purpose, target, and UID are mandatory. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct CredentialSourceRef { + #[schemars( + length(min = 1, max = 253), + regex(pattern = "^kars-credential-source-[a-z0-9][a-z0-9-]*$") + )] + pub name: String, + #[schemars(length(min = 1, max = 128), regex(pattern = "^[A-Za-z0-9-]+$"))] + pub uid: String, +} + +pub const SOURCE_PREFIX: &str = "kars-credential-source-"; +pub const PURPOSE: &str = "kars.azure.com/credential-purpose"; +pub const TARGET: &str = "kars.azure.com/credential-target"; +pub const INTENT: &str = "kars.azure.com/credential-binding-intent"; +pub const SANDBOX_UID: &str = "kars.azure.com/credential-sandbox-uid"; +pub const WORKSPACE: &str = "kars.azure.com/credential-workspace"; +pub const NAMESPACE_UID: &str = "kars.azure.com/credential-namespace-uid"; +pub const SOURCE_UID: &str = "kars.azure.com/credential-source-uid"; +pub const PROJECTION_UID: &str = "kars.azure.com/credential-projection-uid"; +pub const SOURCE_PURPOSE: &str = "agent-source-v1"; +pub const PROJECTION_PURPOSE: &str = "agent-projection-v1"; +pub const BINDING_INTENT: &str = "explicit-reference-v1"; +pub const POD_VERSION: &str = "kars.azure.com/credential-projection-version"; + +// Same agent-only allowlist as the existing handoff credential transport. +// OPENAI_API_KEY and other inference/control-plane variables are deliberately +// absent; those remain router/provider configuration, not agent credentials. +pub const AGENT_KEYS: &[&str] = &[ + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_ALLOW_FROM", + "SLACK_BOT_TOKEN", + "DISCORD_BOT_TOKEN", + "WHATSAPP_ENABLED", + "BRAVE_API_KEY", + "TAVILY_API_KEY", + "EXA_API_KEY", + "FIRECRAWL_API_KEY", + "PERPLEXITY_API_KEY", +]; + +pub fn source_name(sandbox: &str) -> String { + format!("{SOURCE_PREFIX}{sandbox}") +} + +pub fn projection_name(sandbox: &str) -> String { + format!("{sandbox}-credential-projection") +} + +pub fn valid_ref(reference: &CredentialSourceRef, sandbox: &str) -> bool { + reference.name == source_name(sandbox) + && !reference.uid.is_empty() + && reference.uid.len() <= 128 + && reference + .uid + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::crd::KarsSandbox; + use kube::CustomResourceExt; + + #[test] + fn reference_identity_is_retained_in_serialized_spec_digests() { + let mut spec = crate::crd::KarsSandboxSpec { + credentials_ref: Some(CredentialSourceRef { + name: source_name("demo"), + uid: "source-a".into(), + }), + ..Default::default() + }; + let first = crate::providers::signing::sha256_hex(&serde_json::to_vec(&spec).unwrap()); + spec.credentials_ref.as_mut().unwrap().uid = "source-b".into(); + let second = crate::providers::signing::sha256_hex(&serde_json::to_vec(&spec).unwrap()); + assert_ne!(first, second); + } + + #[test] + fn reserved_reference_is_uid_pinned_and_not_an_arbitrary_secret_path() { + let mut reference = CredentialSourceRef { + name: source_name("demo"), + uid: "source-uid".into(), + }; + assert!(valid_ref(&reference, "demo")); + assert!(!valid_ref(&reference, "other")); + reference.name = "controller-receipt-identity".into(); + assert!(!valid_ref(&reference, "demo")); + reference.name = source_name("demo"); + reference.uid = "../source".into(); + assert!(!valid_ref(&reference, "demo")); + } + + #[test] + fn generated_schema_keeps_legacy_specs_unchanged_and_requires_both_reference_fields() { + let spec = crate::crd::KarsSandboxSpec::default(); + assert!( + serde_json::to_value(spec) + .unwrap() + .get("credentialsRef") + .is_none() + ); + let crd = serde_json::to_value(KarsSandbox::crd()).unwrap(); + let schema = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let required = schema["properties"]["credentialsRef"]["required"] + .as_array() + .unwrap(); + assert!(required.contains(&serde_json::json!("name"))); + assert!(required.contains(&serde_json::json!("uid"))); + assert!( + !schema["required"] + .as_array() + .unwrap() + .contains(&serde_json::json!("credentialsRef")) + ); + } +} diff --git a/controller/src/inference_policy_finalizer_tests.rs b/controller/src/inference_policy_finalizer_tests.rs new file mode 100644 index 00000000..d66d7835 --- /dev/null +++ b/controller/src/inference_policy_finalizer_tests.rs @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use serde_json::Value; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, +}; + +const POLICY_PATH: &str = + "/apis/kars.azure.com/v1alpha1/namespaces/workspace/inferencepolicies/example"; +const CONFIGMAP_PATH: &str = + "/api/v1/namespaces/workspace/configmaps/inferencepolicy-example-profile"; +const CUSTOMER_FINALIZER: &str = "customer.example/cleanup"; + +fn policy(deleting: bool) -> InferencePolicy { + let mut value = json!({ + "apiVersion": "kars.azure.com/v1alpha1", "kind": "InferencePolicy", + "metadata": { + "name": "example", "namespace": "workspace", "uid": "policy-uid", + "resourceVersion": "12", "finalizers": [CUSTOMER_FINALIZER], + }, + "spec": { "appliesTo": { "sandboxName": "agent" } }, + }); + if deleting { + value["metadata"]["deletionTimestamp"] = json!("2026-09-07T22:52:35Z"); + value["metadata"]["finalizers"] = json!([FINALIZER, CUSTOMER_FINALIZER]); + } + serde_json::from_value(value).unwrap() +} + +fn status(code: u16) -> ResponseTemplate { + ResponseTemplate::new(code).set_body_json(json!({ + "apiVersion": "v1", "kind": "Status", + "status": if code < 400 { "Success" } else { "Failure" }, + "reason": match code { + 404 => "NotFound", 409 => "Conflict", 403 => "Forbidden", _ => "InternalError" + }, + "code": code, + })) +} + +async fn setup() -> (MockServer, Arc) { + let server = MockServer::start().await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let client = Client::try_from(kube::Config::new(server.uri().parse().unwrap())).unwrap(); + let ctx = Arc::new(Ctx { + client: client.clone(), + http: reqwest::Client::new(), + phase_reporter: PhaseEventReporter::new(client, "InferencePolicy"), + }); + (server, ctx) +} + +async fn expect_patch(server: &MockServer, policy: &InferencePolicy, finalizers: Vec) { + let expected = json!({"metadata": { + "name": "example", "namespace": "workspace", "uid": "policy-uid", + "resourceVersion": "12", "finalizers": finalizers, + }}); + let mut response = policy.clone(); + response.metadata.resource_version = Some("13".into()); + response.metadata.finalizers = Some(finalizers); + Mock::given(method("PATCH")) + .and(path(POLICY_PATH)) + .and(wiremock::matchers::header( + "content-type", + "application/merge-patch+json", + )) + .and(wiremock::matchers::body_json(expected)) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .expect(1) + .mount(server) + .await; +} + +#[tokio::test] +async fn finalizer_registration_preserves_customer_metadata_without_partial_policy_apply() { + let policy = policy(false); + let (server, ctx) = setup().await; + expect_patch( + &server, + &policy, + vec![CUSTOMER_FINALIZER.into(), FINALIZER.into()], + ) + .await; + assert_eq!( + reconcile(Arc::new(policy), ctx).await.unwrap(), + Action::requeue(Duration::from_secs(1)) + ); + assert_eq!(server.received_requests().await.unwrap().len(), 1); +} + +#[tokio::test] +async fn deletion_removes_only_own_finalizer_after_profile_cleanup_or_notfound() { + for code in [200, 404] { + let policy = policy(true); + let (server, ctx) = setup().await; + Mock::given(method("DELETE")) + .and(path(CONFIGMAP_PATH)) + .respond_with(status(code)) + .expect(1) + .mount(&server) + .await; + expect_patch(&server, &policy, vec![CUSTOMER_FINALIZER.into()]).await; + assert_eq!( + reconcile(Arc::new(policy), ctx).await.unwrap(), + Action::await_change() + ); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method, "DELETE"); + assert_eq!(requests[1].method, "PATCH"); + assert!( + requests[1] + .url + .query_pairs() + .all(|(key, value)| key != "force" || value != "true") + ); + } +} + +#[tokio::test] +async fn profile_cleanup_errors_keep_finalizer_and_use_error_retry() { + for code in [403, 409, 500] { + let policy = Arc::new(policy(true)); + let (server, ctx) = setup().await; + Mock::given(method("DELETE")) + .and(path(CONFIGMAP_PATH)) + .respond_with(status(code)) + .expect(1) + .mount(&server) + .await; + let error = reconcile(policy.clone(), ctx.clone()).await.unwrap_err(); + assert!(matches!(&error, ReconcileError::Kube(kube::Error::Api(s)) if s.code == code)); + assert_ne!( + error_policy(policy.clone(), &error, ctx), + Action::await_change() + ); + assert!( + policy + .metadata + .finalizers + .as_ref() + .unwrap() + .contains(&FINALIZER.into()) + ); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } +} + +#[tokio::test] +async fn finalizer_update_conflicts_do_not_force_or_retry_stale_identity() { + for deleting in [false, true] { + let policy = Arc::new(policy(deleting)); + let (server, ctx) = setup().await; + if deleting { + Mock::given(method("DELETE")) + .and(path(CONFIGMAP_PATH)) + .respond_with(status(404)) + .expect(1) + .mount(&server) + .await; + } + Mock::given(method("PATCH")) + .and(path(POLICY_PATH)) + .respond_with(status(409)) + .expect(1) + .mount(&server) + .await; + let error = reconcile(policy.clone(), ctx.clone()).await.unwrap_err(); + assert_ne!(error_policy(policy, &error, ctx), Action::await_change()); + let requests = server.received_requests().await.unwrap(); + let patch = requests.last().unwrap(); + assert_eq!( + patch.headers.get("content-type").unwrap(), + "application/merge-patch+json" + ); + let body: Value = serde_json::from_slice(&patch.body).unwrap(); + assert_eq!(body["metadata"]["uid"], "policy-uid"); + assert_eq!(body["metadata"]["resourceVersion"], "12"); + assert!( + patch + .url + .query_pairs() + .all(|(key, value)| key != "force" || value != "true") + ); + assert_eq!(requests.len(), if deleting { 2 } else { 1 }); + } +} + +#[tokio::test] +async fn finalizer_patch_refuses_missing_api_identity_without_writing() { + for field in ["name", "namespace", "uid", "resourceVersion"] { + let (server, ctx) = setup().await; + let mut value = serde_json::to_value(policy(false)).unwrap(); + value["metadata"].as_object_mut().unwrap().remove(field); + let policy: InferencePolicy = serde_json::from_value(value).unwrap(); + let api = Api::namespaced(ctx.client.clone(), "workspace"); + let result = patch_finalizers(&api, &policy, vec![FINALIZER.into()]).await; + assert!(matches!(result, Err(ReconcileError::MissingIdentity(_)))); + assert!(server.received_requests().await.unwrap().is_empty()); + } +} diff --git a/controller/src/inference_policy_reconciler.rs b/controller/src/inference_policy_reconciler.rs index 5750d1ed..7cd4c67a 100644 --- a/controller/src/inference_policy_reconciler.rs +++ b/controller/src/inference_policy_reconciler.rs @@ -86,6 +86,8 @@ enum ReconcileError { Kube(#[from] kube::Error), #[error("JSON serialization error: {0}")] SerdeJson(#[from] serde_json::Error), + #[error("InferencePolicy metadata is missing {0}")] + MissingIdentity(&'static str), } impl ReconcileError { @@ -95,6 +97,7 @@ impl ReconcileError { match self { ReconcileError::Kube(_) => "kube_api", ReconcileError::SerdeJson(_) => "serde", + ReconcileError::MissingIdentity(_) => "identity", } } } @@ -124,13 +127,9 @@ async fn reconcile(policy: Arc, ctx: Arc) -> Result, + policy: &InferencePolicy, + finalizers: Vec, +) -> Result<(), ReconcileError> { + let required = |value: Option<&str>, field| { + value + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .ok_or(ReconcileError::MissingIdentity(field)) + }; + let name = required(policy.metadata.name.as_deref(), "name")?; + let namespace = required(policy.metadata.namespace.as_deref(), "namespace")?; + let uid = required(policy.metadata.uid.as_deref(), "UID")?; + let resource_version = required( + policy.metadata.resource_version.as_deref(), + "resourceVersion", + )?; + // Finalizers are metadata, not an apply-owned partial policy. Fence the + // update so a stale watch event cannot remove another writer's finalizers. + api.patch( + &name, + &PatchParams::default(), + &Patch::Merge(json!({"metadata": { + "name": name, "namespace": namespace, "uid": uid, + "resourceVersion": resource_version, "finalizers": finalizers + }})), + ) + .await?; + Ok(()) +} + async fn finalize( api: &Api, configmaps: &Api, @@ -773,7 +804,7 @@ async fn finalize( name: &str, ) -> Result { let cm_name = format!("inferencepolicy-{name}-profile"); - let _ = configmaps + configmaps .delete(&cm_name, &Default::default()) .await .map(|_| ()) @@ -783,7 +814,7 @@ async fn finalize( } else { Err(e) } - }); + })?; let finalizers: Vec = policy .metadata @@ -791,13 +822,7 @@ async fn finalize( .as_ref() .map(|v| v.iter().filter(|f| *f != FINALIZER).cloned().collect()) .unwrap_or_default(); - let patch = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"InferencePolicy","metadata":{"finalizers": finalizers}}); - api.patch( - name, - &PatchParams::apply(FIELD_MANAGER).force(), - &Patch::Apply(patch), - ) - .await?; + patch_finalizers(api, policy, finalizers).await?; tracing::info!(inferencepolicy = %name, "InferencePolicyDeleted"); Ok(Action::await_change()) } @@ -878,6 +903,10 @@ pub async fn run(client: Client) -> Result<()> { Ok(()) } +#[cfg(test)] +#[path = "inference_policy_finalizer_tests.rs"] +mod finalizer_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/controller/src/main.rs b/controller/src/main.rs index 0ec22660..6a8a9bb4 100644 --- a/controller/src/main.rs +++ b/controller/src/main.rs @@ -27,6 +27,7 @@ mod crd; #[allow(dead_code)] // CRD-installation pipeline (Phase 1 close-out + future kubectl-claw-attest) consumes these helpers. mod crd_validations; +mod credential_source; mod egress_allowlist_compile; mod egress_approval; mod egress_approval_compile; diff --git a/controller/src/reconciler/agent_env.rs b/controller/src/reconciler/agent_env.rs new file mode 100644 index 00000000..b1975df0 --- /dev/null +++ b/controller/src/reconciler/agent_env.rs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::runtime::RuntimeDeploymentPlan; +use serde_json::{Value, json}; + +pub const RESERVED_PREFIXES: &[&str] = &["AGT_", "FOUNDRY_AGENT_", "AZURE_", "IMDS_", "KARS_"]; + +pub fn merge(env: &mut Vec, plan: &RuntimeDeploymentPlan) { + let mut existing: std::collections::HashSet = env + .iter() + .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) + .collect(); + for (key, value) in &plan.runtime_extra_env { + if key.is_empty() + || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + || key.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + tracing::warn!(key = %key, "extraEnv: invalid env var name, skipping"); + continue; + } + if RESERVED_PREFIXES + .iter() + .any(|prefix| key.starts_with(prefix)) + { + tracing::warn!(key = %key, "extraEnv: key uses reserved prefix, skipping"); + continue; + } + if value.contains('\0') { + tracing::warn!(key = %key, "extraEnv: value contains NUL byte, skipping"); + continue; + } + if !existing.insert(key.clone()) { + tracing::debug!(key = %key, "extraEnv: overridden by reconciler, skipping"); + continue; + } + env.push(json!({"name": key, "value": value})); + } + for entry in &plan.raw_env { + let Some(name) = entry.get("name").and_then(|n| n.as_str()) else { + tracing::warn!("rawEnv: entry missing `name`, skipping"); + continue; + }; + if name.is_empty() + || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + || name.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + tracing::warn!(key = %name, "rawEnv: invalid env var name, skipping"); + continue; + } + if RESERVED_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + { + tracing::warn!(key = %name, "rawEnv: key uses reserved prefix, skipping"); + continue; + } + if !existing.insert(name.into()) { + tracing::debug!(key = %name, "rawEnv: overridden by reconciler, skipping"); + continue; + } + env.push(entry.clone()); + } +} diff --git a/controller/src/reconciler/credential_source_projection.rs b/controller/src/reconciler/credential_source_projection.rs new file mode 100644 index 00000000..fed89bf2 --- /dev/null +++ b/controller/src/reconciler/credential_source_projection.rs @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use kube::api::{DeleteParams, Preconditions}; + +fn owner(ns: &Namespace) -> Result { + Ok(OwnerReference { + api_version: "v1".into(), + kind: "Namespace".into(), + name: ns.name_any(), + uid: identity(&ns.metadata)?.0.into(), + controller: Some(true), + block_owner_deletion: Some(false), + }) +} + +fn validate_owner(meta: &ObjectMeta, sandbox: &KarsSandbox, ns: &Namespace) -> Result<(), Error> { + identity(meta)?; + if meta.name.as_deref() != Some(projection_name(&sandbox.name_any()).as_str()) + || meta.namespace.as_deref() != Some(ns.name_any().as_str()) + || meta.deletion_timestamp.is_some() + || annotation(meta, PURPOSE) != Some(PROJECTION_PURPOSE) + || annotation(meta, TARGET) != sandbox.metadata.name.as_deref() + || annotation(meta, WORKSPACE) != sandbox.metadata.namespace.as_deref() + || annotation(meta, SANDBOX_UID) != sandbox.metadata.uid.as_deref() + || annotation(meta, NAMESPACE_UID) != ns.metadata.uid.as_deref() + || meta.owner_references.as_deref() != Some([owner(ns)?].as_slice()) + { + return Err(Error::Invalid( + "credential projection is not owned by this Sandbox/namespace", + )); + } + Ok(()) +} + +pub(super) fn validate( + meta: &ObjectMeta, + sandbox: &KarsSandbox, + ns: &Namespace, +) -> Result<(), Error> { + validate_owner(meta, sandbox, ns)?; + if annotation(meta, PROJECTION_UID) != Some(identity(meta)?.0) { + return Err(Error::Invalid( + "projection incarnation is not sealed to its UID", + )); + } + Ok(()) +} + +async fn seal( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + meta: &ObjectMeta, +) -> Result, Error> { + namespace_current(client, sandbox, ns).await?; + sandbox_current(client, sandbox).await?; + let (uid, rv) = identity(meta)?; + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let sealed = api.patch_metadata(&projection_name(&sandbox.name_any()), &PatchParams::default(), &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv, "annotations": {PROJECTION_UID: uid}} + }))).await.map_err(|e| api_error("seal projection anchor UID", e))?; + validate(&sealed.metadata, sandbox, ns)?; + Ok(sealed) +} + +pub(super) async fn anchor( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + source: &Secret, +) -> Result, Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let name = projection_name(&sandbox.name_any()); + if let Some(existing) = metadata(&api, &name).await? { + validate_owner(&existing.metadata, sandbox, ns)?; + if annotation(&existing.metadata, PROJECTION_UID).is_some() { + validate(&existing.metadata, sandbox, ns)?; + return Ok(existing); + } + let anchor = api + .get(&name) + .await + .map_err(|e| api_error("inspect unfinished projection anchor", e))?; + let authored = anchor + .metadata + .managed_fields + .as_ref() + .is_some_and(|fields| { + fields.iter().any(|field| { + field.manager.as_deref() == Some(crate::field_managers::CLAWSANDBOX) + }) + }); + if identity(&anchor.metadata)? != identity(&existing.metadata)? + || anchor.type_.as_deref() != Some("Opaque") + || anchor.data.as_ref().is_some_and(|data| !data.is_empty()) + || !authored + { + return Err(Error::Invalid( + "unsealed projection is not an empty controller anchor", + )); + } + return seal(client, sandbox, ns, &anchor.metadata).await; + } + namespace_current(client, sandbox, ns).await?; + sandbox_current(client, sandbox).await?; + let anchor: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": { + "name": name, "namespace": ns.name_any(), + "annotations": { + PURPOSE: PROJECTION_PURPOSE, TARGET: sandbox.name_any(), + WORKSPACE: sandbox.namespace(), SANDBOX_UID: sandbox.metadata.uid, + NAMESPACE_UID: ns.metadata.uid, SOURCE_UID: source.metadata.uid + }, + "ownerReferences": [owner(ns)?] + } + })) + .map_err(|_| Error::Invalid("projection metadata serialization failed"))?; + let created = api + .create( + &PostParams { + field_manager: Some(crate::field_managers::CLAWSANDBOX.into()), + ..Default::default() + }, + &anchor, + ) + .await + .map_err(|e| api_error("create projection anchor", e))?; + validate_owner(&created.metadata, sandbox, ns)?; + seal(client, sandbox, ns, &created.metadata).await +} + +pub(super) async fn revoke( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + remove: bool, +) -> Result<(), Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let name = projection_name(&sandbox.name_any()); + let Some(meta) = metadata(&api, &name).await? else { + return Ok(()); + }; + if validate(&meta.metadata, sandbox, ns).is_err() { + // Never clear/delete an unrelated object, even when its reserved name + // collides with a previously used projection. + return Ok(()); + } + namespace_current(client, sandbox, ns).await?; + let existing = api + .get(&name) + .await + .map_err(|e| api_error("read owned projection for revocation", e))?; + if identity(&existing.metadata)? != identity(&meta.metadata)? + || existing.type_.as_deref() != Some("Opaque") + { + return Err(Error::Invalid( + "projection identity/type changed before revocation", + )); + } + let (uid, rv) = identity(&existing.metadata)?; + if remove { + match api + .delete( + &name, + &DeleteParams { + preconditions: Some(Preconditions { + uid: Some(uid.into()), + resource_version: Some(rv.into()), + }), + ..Default::default() + }, + ) + .await + { + Ok(_) => {} + Err(kube::Error::Api(status)) if status.code == 404 => {} + Err(error) => return Err(api_error("remove owned projection", error)), + } + } else if existing.data.as_ref().is_some_and(|data| !data.is_empty()) { + // JSON merge null clears all keys; {} alone would preserve old keys. + api.patch_metadata( + &name, + &PatchParams::default(), + &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv}, "data": null + })), + ) + .await + .map_err(|e| api_error("revoke owned projection", e))?; + } + Ok(()) +} + +pub(super) async fn detach( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, +) -> Result<(), Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let owned = metadata(&api, &projection_name(&sandbox.name_any())) + .await? + .is_some_and(|value| validate(&value.metadata, sandbox, ns).is_ok()); + workloads::pause(client, sandbox, ns, !owned).await?; + if owned { + revoke(client, sandbox, ns, true).await?; + } + Ok(()) +} diff --git a/controller/src/reconciler/credential_source_test_server.rs b/controller/src/reconciler/credential_source_test_server.rs new file mode 100644 index 00000000..9c250668 --- /dev/null +++ b/controller/src/reconciler/credential_source_test_server.rs @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use crate::crd::KarsSandboxSpec; + +pub const SOURCE_PATH: &str = "/api/v1/namespaces/workspace-a/secrets/kars-credential-source-demo"; +pub const TARGETS_PATH: &str = "/api/v1/namespaces/kars-demo/secrets"; +pub const TARGET_PATH: &str = "/api/v1/namespaces/kars-demo/secrets/demo-credential-projection"; +pub const NS_PATH: &str = "/api/v1/namespaces/kars-demo"; +const SANDBOX_PATH: &str = + "/apis/kars.azure.com/v1alpha1/namespaces/workspace-a/karssandboxes/demo"; +const DEPLOYMENT_PATH: &str = "/apis/apps/v1/namespaces/kars-demo/deployments/demo"; + +pub fn sandbox() -> KarsSandbox { + let mut sandbox = KarsSandbox::new("demo", KarsSandboxSpec::default()); + sandbox.metadata.namespace = Some("workspace-a".into()); + sandbox.metadata.uid = Some("sandbox-a".into()); + sandbox.metadata.resource_version = Some("10".into()); + sandbox.metadata.generation = Some(1); + sandbox.spec.credentials_ref = Some(CredentialSourceRef { + name: source_name("demo"), + uid: "source-a".into(), + }); + sandbox.annotations_mut().insert( + super::super::super::namespace_ownership::NAMESPACE_UID.into(), + "namespace-a".into(), + ); + sandbox +} + +pub fn namespace() -> Namespace { + use crate::reconciler::namespace_ownership as ownership; + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Namespace", + "metadata": { + "name": "kars-demo", "uid": "namespace-a", "resourceVersion": "20", + "annotations": { + (ownership::VERSION): "v1", (ownership::SOURCE_NAME): "demo", + (ownership::SOURCE_NAMESPACE): "workspace-a", (ownership::SOURCE_UID): "sandbox-a" + } + } + })) + .unwrap() +} + +pub fn source() -> Secret { + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": { + "name": source_name("demo"), "namespace": "workspace-a", + "uid": "source-a", "resourceVersion": "30", + "annotations": { + PURPOSE: SOURCE_PURPOSE, TARGET: "demo", WORKSPACE: "workspace-a", INTENT: BINDING_INTENT + } + }, + "data": {"TELEGRAM_BOT_TOKEN": ByteString(b"initial".to_vec())} + })).unwrap() +} + +pub fn deployment() -> Deployment { + serde_json::from_value(json!({ + "apiVersion": "apps/v1", "kind": "Deployment", + "metadata": { + "name": "demo", "namespace": "kars-demo", "uid": "deployment-a", "resourceVersion": "40", + "labels": { + "kars.azure.com/sandbox": "demo", "kars.azure.com/component": "sandbox", + "kars.azure.com/parent-namespace": "workspace-a" + }, + "managedFields": [{ + "manager": crate::field_managers::CLAWSANDBOX, "operation": "Apply", "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", "fieldsV1": {"f:spec": {}} + }] + }, + "spec": { + "replicas": 1, "selector": {"matchLabels": {"kars.azure.com/sandbox": "demo"}}, + "template": {"metadata": {}, "spec": {"containers": [{ + "name": "openclaw", "image": "agent:latest", + "envFrom": [{"secretRef": {"name": "demo-credentials", "optional": true}}] + }]}} + } + })).unwrap() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Fault { + None, + ReplaceNamespaceAfterAnchor, + ReplaceProjectionOnWrite, + ChangeSourceAfterAnchor, + FailValueWrite, + CreateConflict, + SourceReadForbidden, + FailSealOnce, +} + +pub struct State { + pub sandbox: KarsSandbox, + pub namespace: Namespace, + pub source: Option, + pub projection: Option, + pub deployment: Option, + pub fault: Fault, + pub successful_value_writes: usize, + revision: usize, +} + +impl State { + pub fn new() -> Self { + Self { + sandbox: sandbox(), + namespace: namespace(), + source: Some(source()), + projection: None, + deployment: Some(deployment()), + fault: Fault::None, + successful_value_writes: 0, + revision: 100, + } + } +} + +fn response(code: u16, body: Value) -> ResponseTemplate { + ResponseTemplate::new(code).set_body_json(body) +} + +fn failure(code: u16) -> ResponseTemplate { + response( + code, + json!({ + "apiVersion": "v1", "kind": "Status", "status": "Failure", "reason": "Failure", "code": code, + "message": "admission body contains secret-never-log-this" + }), + ) +} + +fn output(request: &Request, value: Value) -> ResponseTemplate { + if request + .headers + .get("accept") + .is_some_and(|header| header.to_str().unwrap().contains("PartialObjectMetadata")) + { + response( + 200, + json!({"apiVersion": "meta.k8s.io/v1", "kind": "PartialObjectMetadata", "metadata": value["metadata"]}), + ) + } else { + response(200, value) + } +} + +fn merge(target: &mut Value, patch: Value) { + if let Value::Object(patch) = patch { + if !target.is_object() { + *target = json!({}); + } + for (key, value) in patch { + if value.is_null() { + target.as_object_mut().unwrap().remove(&key); + } else { + merge( + target + .as_object_mut() + .unwrap() + .entry(key) + .or_insert(Value::Null), + value, + ); + } + } + } else { + *target = patch; + } +} + +pub fn nonempty_value_write(request: &Request) -> bool { + if request.method != "PATCH" || request.url.path() != TARGET_PATH { + return false; + } + let body: Value = serde_json::from_slice(&request.body).unwrap(); + body.get("data") + .and_then(Value::as_object) + .is_some_and(|values| values.values().any(|value| !value.is_null())) +} + +#[derive(Clone)] +struct Server(Arc>); + +impl Respond for Server { + fn respond(&self, request: &Request) -> ResponseTemplate { + let mut state = self.0.lock().unwrap(); + let path = request.url.path(); + if request.method == "GET" { + let value = match path { + SANDBOX_PATH => Some(serde_json::to_value(&state.sandbox).unwrap()), + NS_PATH => Some(serde_json::to_value(&state.namespace).unwrap()), + SOURCE_PATH => { + if state.fault == Fault::SourceReadForbidden { + return failure(403); + } + state + .source + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()) + } + TARGET_PATH => state + .projection + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + DEPLOYMENT_PATH => state + .deployment + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + _ => None, + }; + return value.map_or_else(|| failure(404), |value| output(request, value)); + } + let body: Value = serde_json::from_slice(&request.body).unwrap(); + if request.method == "POST" && path == TARGETS_PATH { + if state.fault == Fault::CreateConflict || state.projection.is_some() { + return failure(409); + } + assert!(body.get("data").is_none() && body.get("stringData").is_none()); + let mut value = body; + value["metadata"]["uid"] = json!("projection-uid"); + value["metadata"]["resourceVersion"] = json!("100"); + value["metadata"]["managedFields"] = json!([{ + "manager": crate::field_managers::CLAWSANDBOX, "operation": "Update", + "apiVersion": "v1", "fieldsType": "FieldsV1", "fieldsV1": {"f:metadata": {}} + }]); + state.projection = Some(serde_json::from_value(value.clone()).unwrap()); + match state.fault { + Fault::ReplaceNamespaceAfterAnchor => { + state.namespace.metadata.uid = Some("replacement".into()) + } + Fault::ChangeSourceAfterAnchor => { + state.source.as_mut().unwrap().metadata.resource_version = + Some("changed".into()) + } + _ => {} + } + return output(request, value); + } + if request.method == "DELETE" && path == TARGET_PATH { + let Some(existing) = state.projection.as_ref() else { + return failure(404); + }; + if body["preconditions"]["uid"] != json!(existing.metadata.uid) + || body["preconditions"]["resourceVersion"] + != json!(existing.metadata.resource_version) + { + return failure(409); + } + return response( + 200, + serde_json::to_value(state.projection.take().unwrap()).unwrap(), + ); + } + if request.method == "PATCH" { + let value_write = nonempty_value_write(request); + if path == TARGET_PATH + && body["metadata"]["annotations"] + .get(PROJECTION_UID) + .is_some() + && state.fault == Fault::FailSealOnce + { + state.fault = Fault::None; + return failure(500); + } + if value_write && state.fault == Fault::ReplaceProjectionOnWrite { + state.projection.as_mut().unwrap().metadata.uid = Some("foreign-projection".into()); + state.projection.as_mut().unwrap().metadata.annotations = None; + } + if value_write && state.fault == Fault::FailValueWrite { + return failure(409); + } + let mut existing = match path { + SOURCE_PATH => state + .source + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + TARGET_PATH => state + .projection + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + DEPLOYMENT_PATH => state + .deployment + .as_ref() + .map(|value| serde_json::to_value(value).unwrap()), + _ if path == format!("{SANDBOX_PATH}/status") => { + Some(serde_json::to_value(&state.sandbox).unwrap()) + } + _ => None, + }; + let Some(ref mut value) = existing else { + return failure(404); + }; + if body["metadata"]["uid"] != value["metadata"]["uid"] + || body["metadata"]["resourceVersion"] != value["metadata"]["resourceVersion"] + { + return failure(409); + } + merge(value, body); + state.revision += 1; + value["metadata"]["resourceVersion"] = json!(state.revision.to_string()); + match path { + SOURCE_PATH => state.source = Some(serde_json::from_value(value.clone()).unwrap()), + TARGET_PATH => { + if value_write { + state.successful_value_writes += 1; + } + state.projection = Some(serde_json::from_value(value.clone()).unwrap()); + } + DEPLOYMENT_PATH => { + state.deployment = Some(serde_json::from_value(value.clone()).unwrap()) + } + _ => state.sandbox = serde_json::from_value(value.clone()).unwrap(), + } + return output(request, value.clone()); + } + failure(500) + } +} + +pub async fn setup(state: State) -> (MockServer, Client, Arc>) { + let server = MockServer::start().await; + let state = Arc::new(Mutex::new(state)); + Mock::given(wiremock::matchers::any()) + .respond_with(Server(state.clone())) + .mount(&server) + .await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let client = Client::try_from(kube::Config::new(server.uri().parse().unwrap())).unwrap(); + (server, client, state) +} + +pub fn resume(state: &Arc>, mode: &Mode) { + let mut state = state.lock().unwrap(); + let sandbox = state.sandbox.clone(); + let ns = state.namespace.clone(); + let deployment = state.deployment.as_mut().unwrap(); + deployment.spec.as_mut().unwrap().replicas = Some(1); + mode.decorate(deployment, &sandbox, &ns); +} diff --git a/controller/src/reconciler/credential_source_tests.rs b/controller/src/reconciler/credential_source_tests.rs new file mode 100644 index 00000000..f1fb1b5c --- /dev/null +++ b/controller/src/reconciler/credential_source_tests.rs @@ -0,0 +1,701 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use std::sync::{Arc, Mutex}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + +#[path = "credential_source_test_server.rs"] +mod server; +use server::*; + +#[test] +fn provider_control_plane_and_process_environment_keys_are_not_credential_sources() { + for key in [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "AZURE_CLIENT_SECRET", + "AGT_SIGNING_KEY", + "KARS_ADMIN_TOKEN", + "NODE_OPTIONS", + "PATH", + "token", + "CUSTOM_TOKEN", + ] { + let mut secret = source(); + secret.data = Some(BTreeMap::from([( + key.into(), + ByteString(b"sensitive".to_vec()), + )])); + assert!(validate_values(&secret).is_err(), "{key}"); + } + for bytes in [vec![0], vec![255]] { + let mut secret = source(); + secret.data = Some(BTreeMap::from([( + "TELEGRAM_BOT_TOKEN".into(), + ByteString(bytes), + )])); + assert!(validate_values(&secret).is_err()); + } +} + +#[test] +fn mode_preserves_legacy_shape_and_keeps_projection_values_out_of_pod_specs() { + assert_eq!( + Mode::Legacy.env_from("demo"), + json!([ + {"secretRef": {"name": "demo-credentials", "optional": true}} + ]) + ); + let mut deployment = deployment(); + let original = deployment.clone(); + Mode::Legacy.decorate(&mut deployment, &sandbox(), &namespace()); + assert_eq!(deployment, original); + let mode = Mode::Source { + uid: "projection-uid".into(), + version: "42".into(), + source_uid: "source-a".into(), + source_version: "30".into(), + }; + mode.decorate(&mut deployment, &sandbox(), &namespace()); + assert_eq!( + mode.env_from("demo"), + json!([ + {"secretRef": {"name": "demo-credential-projection", "optional": false}} + ]) + ); + assert_eq!( + deployment.spec.unwrap().strategy.unwrap().type_.as_deref(), + Some("Recreate") + ); +} + +#[test] +fn errors_never_echo_api_bodies_or_credential_values() { + let error = api_error( + "write", + kube::Error::Api(Box::new( + serde_json::from_value(json!({ + "status": "Failure", "reason": "Invalid", "code": 422, + "message": "stringData: secret-never-log-this" + })) + .unwrap(), + )), + ); + assert!(!format!("{error:?} {error}").contains("secret-never-log-this")); +} + +#[test] +fn metadata_watch_owner_is_the_explicitly_bound_sandbox_uid() { + let reference = source_owner(&sandbox()).unwrap(); + assert_eq!(reference.name, "demo"); + assert_eq!(reference.uid, "sandbox-a"); + assert_eq!(reference.controller, Some(true)); +} + +#[tokio::test] +async fn absent_reference_leaves_legacy_credentials_and_workloads_unchanged() { + let mut state = State::new(); + state.sandbox.spec.credentials_ref = None; + let original = state.deployment.clone(); + let sandbox = state.sandbox.clone(); + let (server, client, state) = setup(state).await; + assert!(matches!( + reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .unwrap(), + Mode::Legacy + )); + assert_eq!(state.lock().unwrap().deployment, original); + assert!( + server + .received_requests() + .await + .unwrap() + .iter() + .all(|r| r.method == "GET") + ); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(|r| r.url.path() == SOURCE_PATH) + ); +} + +#[tokio::test] +async fn first_source_creation_binds_uid_then_fences_values_behind_an_empty_anchor() { + let (server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert!(matches!(mode, Mode::Source { .. })); + let requests = server.received_requests().await.unwrap(); + let anchor = requests + .iter() + .position(|r| r.method == "POST" && r.url.path() == TARGETS_PATH) + .unwrap(); + let anchor_body: Value = serde_json::from_slice(&requests[anchor].body).unwrap(); + assert!(anchor_body.get("data").is_none() && anchor_body.get("stringData").is_none()); + let value_write = requests.iter().position(nonempty_value_write).unwrap(); + assert!(anchor < value_write); + assert!( + requests[anchor + 1..value_write] + .iter() + .any(|r| r.url.path() == NS_PATH) + ); + assert!( + requests[anchor + 1..value_write] + .iter() + .any(|r| r.url.path() == SOURCE_PATH) + ); + let body: Value = serde_json::from_slice(&requests[value_write].body).unwrap(); + assert_eq!(body["metadata"]["uid"], "projection-uid"); + assert!(body["metadata"]["resourceVersion"].is_string()); + let state = state.lock().unwrap(); + assert_eq!( + state.projection.as_ref().unwrap().data, + state.source.as_ref().unwrap().data + ); + assert_eq!( + state + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); + assert_eq!( + annotation(&state.source.as_ref().unwrap().metadata, SANDBOX_UID), + Some("sandbox-a") + ); +} + +#[tokio::test] +async fn source_key_updates_and_removal_refresh_without_resurrecting_legacy_values() { + let (_server, client, state) = setup(State::new()).await; + let first = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &first); + { + let mut state = state.lock().unwrap(); + let source = state.source.as_mut().unwrap(); + source.data = Some(BTreeMap::from([( + "SLACK_BOT_TOKEN".into(), + ByteString(b"rotated".to_vec()), + )])); + source.metadata.resource_version = Some("source-rotated".into()); + } + let next = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert_ne!(format!("{first:?}"), format!("{next:?}")); + let state = state.lock().unwrap(); + let data = state.projection.as_ref().unwrap().data.as_ref().unwrap(); + assert!(!data.contains_key("TELEGRAM_BOT_TOKEN")); + assert_eq!(data["SLACK_BOT_TOKEN"].0, b"rotated"); + assert_eq!( + state + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); +} + +#[tokio::test] +async fn metadata_only_source_changes_do_not_roll_pods_or_rewrite_projection_values() { + let (server, client, state) = setup(State::new()).await; + let first = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &first); + state + .lock() + .unwrap() + .source + .as_mut() + .unwrap() + .metadata + .resource_version = Some("metadata-change".into()); + let before = server.received_requests().await.unwrap().len(); + let next = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + let mut before_deployment = deployment(); + let mut after_deployment = deployment(); + first.decorate(&mut before_deployment, &sandbox(), &namespace()); + next.decorate(&mut after_deployment, &sandbox(), &namespace()); + assert_eq!(before_deployment, after_deployment); + assert!( + server.received_requests().await.unwrap()[before..] + .iter() + .all(|r| r.method == "GET") + ); +} + +#[tokio::test] +async fn missing_replaced_or_invalid_sources_stop_runtime_and_revoke_only_owned_projection() { + for variant in ["missing", "replaced", "purpose", "owner", "provider-key"] { + let (_server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &mode); + { + let mut state = state.lock().unwrap(); + match variant { + "missing" => state.source = None, + "replaced" => { + state.source.as_mut().unwrap().metadata.uid = Some("new-source".into()) + } + "purpose" => { + state + .source + .as_mut() + .unwrap() + .annotations_mut() + .insert(PURPOSE.into(), "other".into()); + } + "owner" => { + state + .source + .as_mut() + .unwrap() + .annotations_mut() + .insert(SANDBOX_UID.into(), "other".into()); + } + _ => { + state.source.as_mut().unwrap().data = Some(BTreeMap::from([( + "OPENAI_API_KEY".into(), + ByteString(b"provider-secret".to_vec()), + )])) + } + } + } + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err(), + "{variant}" + ); + let state = state.lock().unwrap(); + assert!( + state + .projection + .as_ref() + .unwrap() + .data + .as_ref() + .is_none_or(BTreeMap::is_empty), + "{variant}" + ); + assert_eq!( + state + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); + assert_eq!( + state.sandbox.status.as_ref().unwrap().phase.as_deref(), + Some("Degraded") + ); + } +} + +#[tokio::test] +async fn explicit_reference_removal_revokes_projection_and_preserves_legacy_secret() { + let (_server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &mode); + let current = { + let mut state = state.lock().unwrap(); + state.sandbox.spec.credentials_ref = None; + state.sandbox.clone() + }; + assert!(matches!( + reconcile(&client, ¤t, Some(&namespace()), "agent:latest") + .await + .unwrap(), + Mode::Legacy + )); + assert!(state.lock().unwrap().projection.is_none()); + assert_eq!( + state + .lock() + .unwrap() + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); +} + +#[tokio::test] +async fn foreign_projection_and_namespace_never_receive_values_or_get_taken_over() { + for variant in [ + "projection", + "namespace", + "namespace-replaced", + "source-cross-workspace", + ] { + let mut state = State::new(); + match variant { + "projection" => { + let mut foreign = source(); + foreign.metadata.name = Some(projection_name("demo")); + foreign.metadata.namespace = Some("kars-demo".into()); + foreign.metadata.uid = Some("foreign-projection".into()); + state.projection = Some(foreign); + } + "namespace" => { + state.namespace.annotations_mut().insert( + super::super::namespace_ownership::SOURCE_NAMESPACE.into(), + "other-workspace".into(), + ); + } + "namespace-replaced" => state.namespace.metadata.uid = Some("replacement".into()), + _ => state.source.as_mut().unwrap().metadata.namespace = Some("other-workspace".into()), + } + let original = state.projection.clone(); + let (server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(nonempty_value_write) + ); + if variant == "projection" { + assert_eq!(state.lock().unwrap().projection, original); + } + } +} + +#[tokio::test] +async fn namespace_source_and_destination_races_never_write_values_after_failed_fences() { + for fault in [ + Fault::ReplaceNamespaceAfterAnchor, + Fault::ReplaceProjectionOnWrite, + Fault::ChangeSourceAfterAnchor, + Fault::FailValueWrite, + Fault::CreateConflict, + Fault::SourceReadForbidden, + ] { + let mut state = State::new(); + state.fault = fault; + let (_server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert_eq!(state.lock().unwrap().successful_value_writes, 0); + } +} + +#[test] +fn chart_schema_and_generated_reference_contract_agree() { + use kube::CustomResourceExt; + use serde::Deserialize; + let document = serde_yaml::Deserializer::from_str(include_str!( + "../../../deploy/helm/kars/templates/crd.yaml" + )) + .next() + .unwrap(); + let chart = serde_yaml::Value::deserialize(document).unwrap(); + let chart = serde_json::to_value(chart).unwrap(); + let generated = serde_json::to_value(KarsSandbox::crd()).unwrap(); + let path = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/credentialsRef"; + for key in ["name", "uid"] { + for attribute in ["type", "minLength", "maxLength", "pattern"] { + assert_eq!( + chart.pointer(path).unwrap()["properties"][key][attribute], + generated.pointer(path).unwrap()["properties"][key][attribute], + "{key}/{attribute}" + ); + } + } +} + +#[tokio::test] +async fn every_wired_managed_runtime_accepts_the_same_agent_projection() { + for (kind, variant, config) in [ + ("OpenClaw", "openclaw", json!({})), + ("OpenAIAgents", "openaiAgents", json!({})), + ( + "MicrosoftAgentFramework", + "microsoftAgentFramework", + json!({"language": "python"}), + ), + ("LangGraph", "langGraph", json!({"language": "python"})), + ("LangGraph", "langGraph", json!({"language": "typescript"})), + ("Anthropic", "anthropic", json!({})), + ("PydanticAi", "pydanticAi", json!({})), + ("Hermes", "hermes", json!({})), + ( + "BYO", + "byo", + json!({"image": "example.test/agent:latest", "contractVersion": "v1"}), + ), + ] { + let mut state = State::new(); + let mut runtime = json!({"kind": kind}); + runtime[variant] = config; + state.sandbox.spec.runtime = serde_json::from_value(runtime).unwrap(); + let sandbox = state.sandbox.clone(); + let (_server, client, _) = setup(state).await; + let mode = reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert_eq!( + mode.env_from("demo"), + json!([ + {"secretRef": {"name": "demo-credential-projection", "optional": false}} + ]), + "{kind}" + ); + } +} + +#[tokio::test] +async fn empty_source_removes_all_owned_keys_and_does_not_cause_repeated_rollouts() { + let (server, client, state) = setup(State::new()).await; + let first = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &first); + { + let mut state = state.lock().unwrap(); + state.source.as_mut().unwrap().data = None; + state.source.as_mut().unwrap().metadata.resource_version = Some("empty-source".into()); + } + let empty = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert!( + state + .lock() + .unwrap() + .projection + .as_ref() + .unwrap() + .data + .as_ref() + .is_none_or(BTreeMap::is_empty) + ); + resume(&state, &empty); + let before = server.received_requests().await.unwrap().len(); + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert!( + server.received_requests().await.unwrap()[before..] + .iter() + .all(|request| request.method == "GET") + ); +} + +#[tokio::test] +async fn unsafe_reference_names_are_rejected_without_reading_the_named_secret() { + for name in [ + "controller-receipt-identity", + "../other", + "kars-credential-source-other", + ] { + let mut state = State::new(); + state.sandbox.spec.credentials_ref.as_mut().unwrap().name = name.into(); + let sandbox = state.sandbox.clone(); + let (server, client, _) = setup(state).await; + assert!( + reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(|request| request.url.path() == SOURCE_PATH) + ); + } +} + +#[tokio::test] +async fn opaque_type_and_explicit_runtime_env_conflicts_fail_closed_before_projection() { + for variant in ["token-type", "runtime-override", "overlay"] { + let mut state = State::new(); + match variant { + "token-type" => { + state.source.as_mut().unwrap().type_ = + Some("kubernetes.io/service-account-token".into()) + } + "runtime-override" => { + state + .sandbox + .spec + .runtime + .openclaw + .as_mut() + .unwrap() + .extra_env = Some(BTreeMap::from([( + "TELEGRAM_BOT_TOKEN".into(), + "old-value".into(), + )])); + } + _ => { + state.sandbox.spec.upstream_compatibility = Some( + serde_json::from_value(json!({ + "sigsAgentSandbox": "overlay", "upstreamSandboxName": "external" + })) + .unwrap(), + ) + } + } + let sandbox = state.sandbox.clone(); + let (server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox, Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert!(state.lock().unwrap().projection.is_none()); + assert!( + !server + .received_requests() + .await + .unwrap() + .iter() + .any(nonempty_value_write) + ); + } +} + +#[test] +fn extracted_environment_merge_retains_existing_reserved_prefix_and_precedence_rules() { + let runtime = serde_json::from_value(json!({ + "kind": "OpenClaw", "openclaw": {"extraEnv": { + "AGT_KEY": "blocked", "AZURE_CLIENT_SECRET": "blocked", "KARS_TOKEN": "blocked", + "EXISTING": "ignored", "SAFE": "allowed", "NUL": "bad\u{0000}value", "1BAD": "invalid" + }} + })) + .unwrap(); + let mut plan = super::super::runtime::build_runtime_plan(&runtime, "agent:latest").unwrap(); + plan.raw_env = vec![ + json!({"name": "AGT_OTHER", "value": "blocked"}), + json!({"name": "SAFE", "value": "ignored"}), + json!({"name": "FROM_SECRET", "valueFrom": {"secretKeyRef": {"name": "owned", "key": "key"}}}), + ]; + let mut env = vec![json!({"name": "EXISTING", "value": "controller"})]; + super::super::agent_env::merge(&mut env, &plan); + assert_eq!( + env, + vec![ + json!({"name": "EXISTING", "value": "controller"}), + json!({"name": "SAFE", "value": "allowed"}), + json!({"name": "FROM_SECRET", "valueFrom": {"secretKeyRef": {"name": "owned", "key": "key"}}}), + ] + ); +} + +#[test] +fn credential_status_acknowledges_metadata_versions_without_containing_values() { + let mode = Mode::Source { + uid: "projection".into(), + version: "100".into(), + source_uid: "source-a".into(), + source_version: "101".into(), + }; + let mut sandbox = sandbox(); + assert!(mode.needs_status_update(&sandbox)); + let condition = mode.condition(&sandbox).unwrap(); + assert_eq!(condition.type_, "CredentialsReady"); + assert!(condition.message.contains("sourceVersion")); + assert!(!condition.message.contains("initial")); + sandbox.status = Some(crate::crd::KarsSandboxStatus { + conditions: vec![condition], + ..Default::default() + }); + assert!(!mode.needs_status_update(&sandbox)); + assert!(Mode::Legacy.needs_status_update(&sandbox)); +} + +#[tokio::test] +async fn a_recreated_projection_cannot_inherit_the_previous_uid_seal() { + let (_server, client, state) = setup(State::new()).await; + let mode = reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .unwrap(); + resume(&state, &mode); + let foreign = { + let mut state = state.lock().unwrap(); + state.projection.as_mut().unwrap().metadata.uid = Some("replacement-uid".into()); + state.projection.clone() + }; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert_eq!(state.lock().unwrap().projection, foreign); + assert_eq!( + state + .lock() + .unwrap() + .deployment + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .replicas, + Some(0) + ); +} + +#[tokio::test] +async fn an_empty_controller_anchor_recovers_after_a_failed_uid_seal() { + let mut state = State::new(); + state.fault = Fault::FailSealOnce; + let (_server, client, state) = setup(state).await; + assert!( + reconcile(&client, &sandbox(), Some(&namespace()), "agent:latest") + .await + .is_err() + ); + assert_eq!(state.lock().unwrap().successful_value_writes, 0); + let current = state.lock().unwrap().sandbox.clone(); + reconcile(&client, ¤t, Some(&namespace()), "agent:latest") + .await + .unwrap(); + assert_eq!(state.lock().unwrap().successful_value_writes, 1); +} diff --git a/controller/src/reconciler/credential_source_workloads.rs b/controller/src/reconciler/credential_source_workloads.rs new file mode 100644 index 00000000..ecf44ac4 --- /dev/null +++ b/controller/src/reconciler/credential_source_workloads.rs @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; + +fn consumer(meta: &ObjectMeta, sandbox: &KarsSandbox, ns: &Namespace) -> bool { + annotation(meta, SANDBOX_UID) == sandbox.metadata.uid.as_deref() + && annotation(meta, NAMESPACE_UID) == ns.metadata.uid.as_deref() +} + +fn owned(meta: &ObjectMeta, sandbox: &KarsSandbox, ns: &Namespace) -> Result<(), Error> { + identity(meta)?; + let labels = meta.labels.as_ref().cloned().unwrap_or_default(); + let authored = meta.managed_fields.as_ref().is_some_and(|fields| { + fields.iter().any(|entry| { + entry.manager.as_deref() == Some(crate::field_managers::CLAWSANDBOX) + && entry.operation.as_deref() == Some("Apply") + && entry + .fields_v1 + .as_ref() + .is_some_and(|fields| fields.0.get("f:spec").is_some()) + }) + }); + if meta.name != sandbox.metadata.name + || meta.namespace.as_deref() != Some(ns.name_any().as_str()) + || meta.deletion_timestamp.is_some() + || meta + .owner_references + .as_ref() + .is_some_and(|refs| !refs.is_empty()) + || labels.get("kars.azure.com/sandbox") != sandbox.metadata.name.as_ref() + || labels.get("kars.azure.com/component").map(String::as_str) != Some("sandbox") + || labels + .get("kars.azure.com/parent-namespace") + .is_some_and(|value| Some(value) != sandbox.metadata.namespace.as_ref()) + || (!authored && !consumer(meta, sandbox, ns)) + { + return Err(Error::Invalid( + "runtime Deployment ownership is unproven; no workload takeover", + )); + } + Ok(()) +} + +pub(super) async fn pause( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + only_consumer: bool, +) -> Result<(), Error> { + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let deployment = match api.get(&sandbox.name_any()).await { + Ok(value) => value, + Err(kube::Error::Api(status)) if status.code == 404 => return Ok(()), + Err(error) => return Err(api_error("read runtime Deployment", error)), + }; + if only_consumer && !consumer(&deployment.metadata, sandbox, ns) { + return Ok(()); + } + owned(&deployment.metadata, sandbox, ns)?; + let strategy = if sandbox.spec.credentials_ref.is_some() { + "Recreate" + } else { + "RollingUpdate" + }; + if deployment.spec.as_ref().and_then(|spec| spec.replicas) == Some(0) + && deployment + .spec + .as_ref() + .and_then(|spec| spec.strategy.as_ref()) + .is_some_and(|value| { + value.type_.as_deref() == Some(strategy) + && (strategy != "Recreate" || value.rolling_update.is_none()) + }) + { + return Ok(()); + } + namespace_current(client, sandbox, ns).await?; + let (uid, rv) = identity(&deployment.metadata)?; + api.patch( + &sandbox.name_any(), + &PatchParams::default(), + &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv}, + "spec": {"replicas": 0, "strategy": {"type": strategy, "rollingUpdate": null}} + })), + ) + .await + .map_err(|e| api_error("stop credential consumer", e))?; + Ok(()) +} + +pub(super) async fn current( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + mode: &Mode, +) -> Result { + let Mode::Source { uid, version, .. } = mode else { + return Ok(false); + }; + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let deployment = match api.get(&sandbox.name_any()).await { + Ok(value) => value, + Err(kube::Error::Api(status)) if status.code == 404 => return Ok(false), + Err(error) => return Err(api_error("read credential consumer", error)), + }; + owned(&deployment.metadata, sandbox, ns)?; + let expected = format!("{uid}:{version}"); + Ok(consumer(&deployment.metadata, sandbox, ns) + && deployment + .spec + .as_ref() + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|meta| annotation(meta, POD_VERSION)) + == Some(expected.as_str())) +} diff --git a/controller/src/reconciler/credential_sources.rs b/controller/src/reconciler/credential_sources.rs new file mode 100644 index 00000000..e6099c53 --- /dev/null +++ b/controller/src/reconciler/credential_sources.rs @@ -0,0 +1,509 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Exclusive, opt-in agent credentials. Values only enter a UID/RV-fenced +//! projection after source, Sandbox, namespace, and target identity checks. + +use crate::{crd::KarsSandbox, credential_source::*}; +use k8s_openapi::{ + ByteString, + api::{ + apps::v1::{Deployment, DeploymentStrategy}, + core::v1::{Namespace, Secret}, + }, + apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}, +}; +use kube::{ + Api, Client, ResourceExt, + api::{Patch, PatchParams, PostParams}, + core::PartialObjectMeta, +}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[path = "credential_source_projection.rs"] +mod projection; +#[path = "credential_source_workloads.rs"] +mod workloads; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("CredentialSourceUnavailable: {0}")] + Invalid(&'static str), + #[error("CredentialSourceUnavailable: {stage} failed (Kubernetes status {code:?})")] + Api { + stage: &'static str, + code: Option, + }, +} + +fn api_error(stage: &'static str, error: kube::Error) -> Error { + Error::Api { + stage, + code: match error { + kube::Error::Api(status) => Some(status.code), + _ => None, + }, + } +} + +fn identity(meta: &ObjectMeta) -> Result<(&str, &str), Error> { + match (meta.uid.as_deref(), meta.resource_version.as_deref()) { + (Some(uid), Some(rv)) if !uid.is_empty() && !rv.is_empty() => Ok((uid, rv)), + _ => Err(Error::Invalid("API object omitted UID/resourceVersion")), + } +} + +fn annotation<'a>(meta: &'a ObjectMeta, key: &str) -> Option<&'a str> { + meta.annotations.as_ref()?.get(key).map(String::as_str) +} + +async fn metadata( + api: &Api, + name: &str, +) -> Result>, Error> { + match api.get_metadata(name).await { + Ok(value) => Ok(Some(value)), + Err(kube::Error::Api(status)) if status.code == 404 => Ok(None), + Err(error) => Err(api_error("read Secret metadata", error)), + } +} + +async fn namespace_current( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, +) -> Result<(), Error> { + super::namespace_ownership::recheck(client, sandbox, ns) + .await + .map_err(|_| Error::Invalid("runtime namespace authority changed"))?; + Ok(()) +} + +async fn sandbox_current(client: &Client, sandbox: &KarsSandbox) -> Result<(), Error> { + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let live = api + .get_metadata(&sandbox.name_any()) + .await + .map_err(|e| api_error("recheck Sandbox", e))?; + if identity(&live.metadata)? != identity(&sandbox.metadata)? + || live.metadata.namespace != sandbox.metadata.namespace + || live.metadata.name != sandbox.metadata.name + || live.metadata.deletion_timestamp.is_some() + { + return Err(Error::Invalid("Sandbox identity/reference changed")); + } + Ok(()) +} + +fn source_owner(sandbox: &KarsSandbox) -> Result { + Ok(OwnerReference { + api_version: "kars.azure.com/v1alpha1".into(), + kind: "KarsSandbox".into(), + name: sandbox.name_any(), + uid: identity(&sandbox.metadata)?.0.into(), + controller: Some(true), + block_owner_deletion: Some(false), + }) +} + +fn source_valid( + meta: &ObjectMeta, + sandbox: &KarsSandbox, + ns: &Namespace, + bound: bool, +) -> Result<(), Error> { + let reference = sandbox + .spec + .credentials_ref + .as_ref() + .ok_or(Error::Invalid("reference absent"))?; + if identity(meta)?.0 != reference.uid + || meta.name.as_deref() != Some(reference.name.as_str()) + || meta.namespace != sandbox.metadata.namespace + || meta.deletion_timestamp.is_some() + || annotation(meta, PURPOSE) != Some(SOURCE_PURPOSE) + || annotation(meta, TARGET) != sandbox.metadata.name.as_deref() + || annotation(meta, WORKSPACE) != sandbox.metadata.namespace.as_deref() + || annotation(meta, INTENT) != Some(BINDING_INTENT) + { + return Err(Error::Invalid( + "source identity, purpose, target, or lifecycle is invalid", + )); + } + let owner = source_owner(sandbox)?; + let refs = meta.owner_references.as_deref().unwrap_or_default(); + if (!refs.is_empty() && refs != [owner.clone()]) || (bound && refs != [owner]) { + return Err(Error::Invalid("source belongs to another owner")); + } + for (key, expected) in [ + (SANDBOX_UID, identity(&sandbox.metadata)?.0), + (NAMESPACE_UID, identity(&ns.metadata)?.0), + ] { + match annotation(meta, key) { + Some(value) if value == expected => {} + None if !bound => {} + _ => { + return Err(Error::Invalid( + "source Sandbox/namespace UID binding is invalid", + )); + } + } + } + Ok(()) +} + +fn validate_values(secret: &Secret) -> Result<(), Error> { + if secret.type_.as_deref() != Some("Opaque") + || secret + .string_data + .as_ref() + .is_some_and(|data| !data.is_empty()) + { + return Err(Error::Invalid("source must be a persisted Opaque Secret")); + } + let mut bytes = 0usize; + for (key, value) in secret.data.iter().flatten() { + if !AGENT_KEYS.contains(&key.as_str()) + || super::agent_env::RESERVED_PREFIXES + .iter() + .any(|prefix| key.starts_with(prefix)) + || value.0.contains(&0) + || std::str::from_utf8(&value.0).is_err() + { + return Err(Error::Invalid( + "source contains disallowed agent keys or invalid environment values", + )); + } + bytes += value.0.len(); + } + if bytes > 131_072 { + return Err(Error::Invalid( + "source credential collection exceeds 128 KiB", + )); + } + Ok(()) +} + +async fn read_source( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + default_image: &str, +) -> Result { + let reference = sandbox + .spec + .credentials_ref + .as_ref() + .ok_or(Error::Invalid("reference absent"))?; + if !valid_ref(reference, &sandbox.name_any()) { + return Err(Error::Invalid( + "reference must pin the reserved source name and exact UID", + )); + } + if sandbox + .spec + .upstream_compatibility + .as_ref() + .is_some_and(|value| value.is_overlay_mode()) + { + return Err(Error::Invalid( + "credential sources require a controller-managed runtime", + )); + } + let plan = + super::runtime::build_runtime_plan(&sandbox.spec.runtime, default_image).map_err(|_| { + Error::Invalid("credential source runtime configuration is invalid or unsupported") + })?; + if AGENT_KEYS + .iter() + .any(|key| plan.runtime_extra_env.contains_key(*key)) + || plan.raw_env.iter().any(|entry| { + entry + .get("name") + .and_then(Value::as_str) + .is_some_and(|key| AGENT_KEYS.contains(&key)) + }) + { + return Err(Error::Invalid( + "source mode cannot be combined with runtime credential env overrides", + )); + } + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let meta = metadata(&api, &reference.name) + .await? + .ok_or(Error::Invalid("source Secret is missing"))?; + source_valid(&meta.metadata, sandbox, ns, false)?; + let mut source = api + .get(&reference.name) + .await + .map_err(|e| api_error("read credential source", e))?; + if identity(&source.metadata)? != identity(&meta.metadata)? { + return Err(Error::Invalid("source changed during read")); + } + source_valid(&source.metadata, sandbox, ns, false)?; + validate_values(&source)?; + if source_valid(&source.metadata, sandbox, ns, true).is_err() { + let (uid, rv) = identity(&source.metadata)?; + let bound = api.patch_metadata(&reference.name, &PatchParams::default(), &Patch::Merge(json!({ + "metadata": { + "uid": uid, "resourceVersion": rv, + "ownerReferences": [source_owner(sandbox)?], + "annotations": {SANDBOX_UID: sandbox.metadata.uid, NAMESPACE_UID: ns.metadata.uid} + } + }))).await.map_err(|e| api_error("bind credential source", e))?; + source.metadata = bound.metadata; + source_valid(&source.metadata, sandbox, ns, true)?; + } + Ok(source) +} + +async fn inputs_current( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + source: &Secret, +) -> Result<(), Error> { + sandbox_current(client, sandbox).await?; + namespace_current(client, sandbox, ns).await?; + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let live = metadata(&api, &source.name_any()) + .await? + .ok_or(Error::Invalid("source disappeared"))?; + source_valid(&live.metadata, sandbox, ns, true)?; + if identity(&live.metadata)? != identity(&source.metadata)? { + return Err(Error::Invalid("source changed before projection write")); + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub enum Mode { + Legacy, + Source { + uid: String, + version: String, + source_uid: String, + source_version: String, + }, +} + +impl Mode { + pub fn env_from(&self, sandbox: &str) -> Value { + match self { + Self::Legacy => { + json!([{"secretRef": {"name": format!("{sandbox}-credentials"), "optional": true}}]) + } + Self::Source { .. } => { + json!([{"secretRef": {"name": projection_name(sandbox), "optional": false}}]) + } + } + } + + pub fn decorate(&self, deployment: &mut Deployment, sandbox: &KarsSandbox, ns: &Namespace) { + if let Self::Source { uid, version, .. } = self { + let annotations = deployment.metadata.annotations.get_or_insert_default(); + annotations.insert( + SANDBOX_UID.into(), + sandbox.metadata.uid.clone().unwrap_or_default(), + ); + annotations.insert( + NAMESPACE_UID.into(), + ns.metadata.uid.clone().unwrap_or_default(), + ); + let spec = deployment + .spec + .as_mut() + .expect("controller builds a Deployment spec"); + spec.strategy = Some(DeploymentStrategy { + type_: Some("Recreate".into()), + rolling_update: None, + }); + spec.template + .metadata + .get_or_insert_default() + .annotations + .get_or_insert_default() + .insert(POD_VERSION.into(), format!("{uid}:{version}")); + } + } + + pub fn condition( + &self, + sandbox: &KarsSandbox, + ) -> Option { + let prior = sandbox.status.as_ref().and_then(|status| { + status + .conditions + .iter() + .find(|condition| condition.type_ == "CredentialsReady") + }); + let (status, reason, message) = match self { + Self::Source { + uid, + version, + source_uid, + source_version, + } => ( + "True", + "Projected", + json!({"sourceUid": source_uid, "sourceVersion": source_version, + "projectionUid": uid, "projectionVersion": version}) + .to_string(), + ), + Self::Legacy if prior.is_some() => ( + "False", + "DirectCredentials", + "Explicit source is not configured".into(), + ), + Self::Legacy => return None, + }; + Some(crate::status::conditions::preserve_transition_time( + prior, + "CredentialsReady", + status, + reason, + &message, + sandbox.metadata.generation, + )) + } + + pub fn needs_status_update(&self, sandbox: &KarsSandbox) -> bool { + self.condition(sandbox).is_some_and(|expected| { + !sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|current| { + current.type_ == expected.type_ + && current.status == expected.status + && current.reason == expected.reason + && current.message == expected.message + }) + }) + }) + } +} + +pub async fn reconcile( + client: &Client, + sandbox: &KarsSandbox, + ns: Option<&Namespace>, + default_image: &str, +) -> Result { + let ns = ns.ok_or(Error::Invalid("runtime namespace is not verified"))?; + let result = if sandbox.spec.credentials_ref.is_some() { + project(client, sandbox, ns, default_image).await + } else { + projection::detach(client, sandbox, ns) + .await + .map(|()| Mode::Legacy) + }; + if let Err(error) = result { + // Try both operations: a transient Deployment error must not skip + // projection revocation, or vice versa. Never echo API request bodies. + let stopped = + workloads::pause(client, sandbox, ns, sandbox.spec.credentials_ref.is_none()).await; + let revoked = projection::revoke(client, sandbox, ns, false).await; + let failure = stopped.err().or_else(|| revoked.err()).unwrap_or(error); + report(client, sandbox, &failure).await?; + return Err(failure); + } + result +} + +async fn project( + client: &Client, + sandbox: &KarsSandbox, + ns: &Namespace, + default_image: &str, +) -> Result { + namespace_current(client, sandbox, ns).await?; + let source = read_source(client, sandbox, ns, default_image).await?; + let target = projection::anchor(client, sandbox, ns, &source).await?; + let api: Api = Api::namespaced(client.clone(), &ns.name_any()); + let current = api + .get(&target.name_any()) + .await + .map_err(|e| api_error("read owned projection", e))?; + projection::validate(¤t.metadata, sandbox, ns)?; + if identity(¤t.metadata)? != identity(&target.metadata)? + || current.type_.as_deref() != Some("Opaque") + || current.immutable == Some(true) + { + return Err(Error::Invalid("projection identity/type changed")); + } + let values: BTreeMap = source.data.clone().unwrap_or_default(); + let changed = current.data.clone().unwrap_or_default() != values + || annotation(¤t.metadata, SOURCE_UID) != source.metadata.uid.as_deref(); + let mut mode = Mode::Source { + uid: identity(¤t.metadata)?.0.into(), + version: identity(¤t.metadata)?.1.into(), + source_uid: identity(&source.metadata)?.0.into(), + source_version: identity(&source.metadata)?.1.into(), + }; + if changed || !workloads::current(client, sandbox, ns, &mode).await? { + workloads::pause(client, sandbox, ns, false).await?; + } + inputs_current(client, sandbox, ns, &source).await?; + if changed { + let (uid, rv) = identity(¤t.metadata)?; + let mut data = serde_json::to_value(&values) + .map_err(|_| Error::Invalid("credential serialization failed"))?; + for key in current.data.iter().flat_map(|data| data.keys()) { + if !values.contains_key(key) { + data[key] = Value::Null; + } + } + let written = api.patch_metadata(&target.name_any(), &PatchParams::default(), &Patch::Merge(json!({ + "metadata": {"uid": uid, "resourceVersion": rv, "annotations": {SOURCE_UID: source.metadata.uid}}, + "data": data + }))).await.map_err(|e| api_error("write credential projection", e))?; + projection::validate(&written.metadata, sandbox, ns)?; + mode = Mode::Source { + uid: identity(&written.metadata)?.0.into(), + version: identity(&written.metadata)?.1.into(), + source_uid: identity(&source.metadata)?.0.into(), + source_version: identity(&source.metadata)?.1.into(), + }; + } + Ok(mode) +} + +async fn report(client: &Client, sandbox: &KarsSandbox, error: &Error) -> Result<(), Error> { + let api: Api = + Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + let live = api + .get(&sandbox.name_any()) + .await + .map_err(|e| api_error("read credential status target", e))?; + if live.metadata.uid != sandbox.metadata.uid { + return Err(Error::Invalid("Sandbox was recreated")); + } + let message = error.to_string(); + let mut patch = + crate::status::build_degraded_status_patch(&live, "CredentialSourceUnavailable", &message); + let existing = serde_json::to_value(&live.status) + .map_err(|_| Error::Invalid("status serialization failed"))?; + if patch["status"].as_object().is_some_and(|fields| { + fields + .iter() + .all(|(key, val)| existing.get(key) == Some(val)) + }) { + return Ok(()); + } + let (uid, rv) = identity(&live.metadata)?; + patch["metadata"] = json!({"uid": uid, "resourceVersion": rv}); + api.patch_status( + &live.name_any(), + &PatchParams::default(), + &Patch::Merge(patch), + ) + .await + .map_err(|e| api_error("report credential failure", e))?; + Ok(()) +} + +#[cfg(test)] +#[path = "credential_source_tests.rs"] +mod tests; diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 38bcdfcd..8e86917f 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -33,7 +33,9 @@ use tokio::time::Duration; use crate::crd::KarsSandbox; use crate::fedcred::{FedCredConfig, FedCredManager}; +mod agent_env; pub(crate) mod byo_contract; +mod credential_sources; mod dev_env; pub(crate) mod governance_mounts; mod inference; @@ -60,6 +62,8 @@ fn sandbox_node_selector(default_pool: &str) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result = openclaw_env - .iter() - .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) - .collect(); - for (k, v) in &runtime_plan.runtime_extra_env { - if k.is_empty() - || !k.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - || k.chars().next().is_some_and(|c| c.is_ascii_digit()) - { - tracing::warn!(key = %k, "extraEnv: invalid env var name, skipping"); - continue; - } - if RESERVED_PREFIXES.iter().any(|p| k.starts_with(p)) { - tracing::warn!(key = %k, "extraEnv: key uses reserved prefix, skipping"); - continue; - } - if v.contains('\0') { - tracing::warn!(key = %k, "extraEnv: value contains NUL byte, skipping"); - continue; - } - if existing.contains(k) { - tracing::debug!(key = %k, "extraEnv: overridden by reconciler, skipping"); - continue; - } - openclaw_env.push(json!({"name": k, "value": v})); - existing.insert(k.clone()); - } - } - - // S10.A2.b: append `plan.raw_env` entries (BYO `valueFrom` etc.). - // The producer guarantees these have a `name` field; we apply the - // same reserved-prefix / NUL / dup filter to the `name` only — - // the `valueFrom` payload itself is rendered verbatim. - if !runtime_plan.raw_env.is_empty() { - const RESERVED_PREFIXES: &[&str] = - &["AGT_", "FOUNDRY_AGENT_", "AZURE_", "IMDS_", "KARS_"]; - let mut existing: std::collections::HashSet = openclaw_env - .iter() - .filter_map(|v| v.get("name").and_then(|n| n.as_str()).map(String::from)) - .collect(); - for entry in &runtime_plan.raw_env { - let Some(name) = entry.get("name").and_then(|n| n.as_str()) else { - tracing::warn!("rawEnv: entry missing `name`, skipping"); - continue; - }; - if name.is_empty() - || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - || name.chars().next().is_some_and(|c| c.is_ascii_digit()) - { - tracing::warn!(key = %name, "rawEnv: invalid env var name, skipping"); - continue; - } - if RESERVED_PREFIXES.iter().any(|p| name.starts_with(p)) { - tracing::warn!(key = %name, "rawEnv: key uses reserved prefix, skipping"); - continue; - } - if existing.contains(name) { - tracing::debug!(key = %name, "rawEnv: overridden by reconciler, skipping"); - continue; - } - openclaw_env.push(entry.clone()); - existing.insert(name.to_string()); - } - } + agent_env::merge(&mut openclaw_env, &runtime_plan); // Build the inference-router env array let mut router_env = vec![ @@ -1929,9 +1872,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result = allowlist_resolution.conditions.clone(); + if let Some(condition) = credentials.condition(&sandbox) { + extras.push(condition); + } // Phase G P1 #4: stamp Suspended condition when spec.suspended // is true, or surface Suspended=False/Active when there is a @@ -3140,7 +3087,8 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); let status_obj = crate::status::build_running_status_patch_with_extras( @@ -3173,7 +3121,13 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result Duration { let base = match error { // Transient kube API errors (throttling, connection reset, 5xx): // retry soon so we don't starve legitimate work. - ReconcileError::Kube(_) | ReconcileError::NamespaceOwnership(_) => 30, + ReconcileError::Kube(_) + | ReconcileError::NamespaceOwnership(_) + | ReconcileError::Credentials(_) => 30, // Serde errors are deterministic — the same body will fail again. // Back off longer so we don't spam logs while a human fixes the // bad CR. @@ -3200,6 +3156,7 @@ fn error_policy(sandbox: Arc, error: &ReconcileError, _ctx: Arc "serde", ReconcileError::Configuration(_) => "configuration", ReconcileError::NamespaceOwnership(_) => "namespace_ownership", + ReconcileError::Credentials(_) => "credentials", }; crate::metrics::record_reconcile_error("KarsSandbox", class); tracing::error!( @@ -3377,6 +3334,10 @@ pub async fn run(client: Client) -> Result<()> { }); Controller::new(sandboxes, crate::watch_config::bounded()) + .owns( + Api::::all(ctx.client.clone()), + crate::watch_config::bounded(), + ) .watches( Api::::all(ctx.client.clone()), crate::watch_config::bounded(), diff --git a/deploy/helm/kars/templates/crd.yaml b/deploy/helm/kars/templates/crd.yaml index 3e9d7506..e146f7e2 100644 --- a/deploy/helm/kars/templates/crd.yaml +++ b/deploy/helm/kars/templates/crd.yaml @@ -15,11 +15,16 @@ spec: schema: openAPIV3Schema: type: object + x-kubernetes-validations: + - rule: "!has(self.spec) || !has(self.spec.credentialsRef) || self.spec.credentialsRef.name == 'kars-credential-source-' + self.metadata.name" + message: "credentialsRef must use the reserved source name for this Sandbox" properties: spec: type: object required: ["runtime", "sandbox", "inferenceRef"] x-kubernetes-validations: + - rule: "!has(self.credentialsRef) || !has(self.upstreamCompatibility) || !has(self.upstreamCompatibility.sigsAgentSandbox) || self.upstreamCompatibility.sigsAgentSandbox != 'overlay'" + message: "credentialsRef requires a controller-managed runtime, not overlay mode" # P2 #13: top-level cross-field invariants. # # 1. BYO runtime is mutually exclusive with the @@ -45,6 +50,41 @@ spec: message: "spec.governance.toolPolicyRef.name must be a same-namespace name (no '/' or ':' separators); cross-namespace refs are forbidden" reason: "FieldValueInvalid" properties: + upstreamCompatibility: + type: object + description: "Existing upstream interoperability settings; omitted by native Sandboxes." + x-kubernetes-validations: + - rule: "!has(self.sigsAgentSandbox) || self.sigsAgentSandbox != 'overlay' || has(self.upstreamSandboxRef)" + message: "overlay mode requires a same-namespace upstreamSandboxRef" + properties: + sigsAgentSandbox: + type: string + enum: ["off", "observe", "translate", "overlay"] + upstreamSandboxRef: + type: object + required: ["name"] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + aiConformanceReference: + type: boolean + credentialsRef: + type: object + description: "Explicit agent credential collection in this Sandbox's workspace; replaces legacy credentials while set. Missing/replaced sources fail closed." + required: ["name", "uid"] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + pattern: "^kars-credential-source-[a-z0-9][a-z0-9-]*$" + uid: + type: string + minLength: 1 + maxLength: 128 + pattern: "^[A-Za-z0-9-]+$" runtime: type: object required: ["kind"] diff --git a/docs/how-to/credential-sources.md b/docs/how-to/credential-sources.md new file mode 100644 index 00000000..4335caec --- /dev/null +++ b/docs/how-to/credential-sources.md @@ -0,0 +1,182 @@ +# Workspace credential sources (v1) + +Credential sources are an **optional, explicit** alternative to the existing +`kars-/-credentials` Secret. Install the controller and CRD containing +this feature before enabling it. Namespace claim v1 is a prerequisite; complete +[namespace ownership preflight/adoption](namespace-ownership.md) first. + +With `spec.credentialsRef` absent, agents retain the existing direct Secret +EnvFrom and defaults. No source is discovered or delivered merely because its +name matches a future Sandbox. + +## CLI workflows + +New Sandbox, with credentials stored through the existing masked local prompt: + +```sh +kars credentials set telegram-token +kars add demo --channels telegram --credential-source +``` + +The CLI creates the workspace source **before** the Sandbox, obtains its real +API-server UID, and includes that UID in the Sandbox CREATE. A create conflict +does not overwrite a racing Sandbox. `--namespace ` selects the +workspace for the source, Sandbox, and companion policies; the default remains +`kars-system`. + +For an existing Sandbox: + +```sh +kars credentials update demo --use-source +kars credentials update demo --telegram-token "$NEW_TOKEN" +kars credentials update demo --remove telegram-token +``` + +`--use-source` migrates the entire direct credential collection once, without +changing the legacy Secret. Explicit updates override imported values. Existing +staged source values override legacy values. Removal is applied last. Migration +requires read access to the verified runtime namespace and its direct Secret; +new source-bound Sandboxes do not require runtime-namespace Secret writes. + +Once bound, ordinary `credentials update` automatically updates the pinned source +in the selected workspace. Pass `--namespace` for Sandboxes outside `kars-system`. +The command requires read access to that Sandbox to choose the correct path. +Direct updates also inspect runtime namespace claim metadata when present, so a +wrong workspace or same-name conflict cannot be mistaken for a direct target. +Source updates and removals use UID/resourceVersion checks and never +delete/recreate the source. `--remove` accepts comma-separated environment keys +or credential flag names. The existing `credentials remove ` command still +removes a **local** stored credential. + +Source mode always refreshes the runtime. `--no-restart` remains available for +direct credentials but is rejected for source changes. Source-mode operations +wait for the controller to acknowledge the current source UID/resourceVersion +in `CredentialsReady`; they do not claim success merely because an older +controller accepted a CR. API schema pruning is detected after submission. +Existing secret-value flags retain their usual shell/process-argument exposure; +the source integration passes Kubernetes Secret bodies through stdin and does +not print their values. + +Explicit opt-out: + +```sh +kars credentials update demo --disable-source +``` + +The controller stops the previous credential consumer, removes **only its owned +projection**, and returns to the unchanged direct Secret. This intentionally +reactivates the legacy collection: review it before opting out. The workspace +source remains available for the same Sandbox UID; it is not Helm-owned. + +Explicit pre-CR storage is supported: + +```sh +kars credentials update demo --use-source --telegram-token "$TOKEN" +kars add demo --credential-source +``` + +The first command stores an opted-in source, not a delivery instruction. The +second explicitly selects its UID. Abandoned unbound sources can have their +keys removed with `credentials update --use-source --remove`; normal Kubernetes +Secret deletion can remove the empty resource. Bound sources have a same- +workspace Sandbox owner reference and are garbage-collected with that Sandbox. +A different Sandbox UID cannot inherit a previous binding. + +## API contract + +The source is an Opaque Secret in the **same namespace as the KarsSandbox CR**: + +* Name: `kars-credential-source-`. +* Annotations: + * `kars.azure.com/credential-purpose: agent-source-v1` + * `kars.azure.com/credential-target: ` + * `kars.azure.com/credential-workspace: ` + * `kars.azure.com/credential-binding-intent: explicit-reference-v1` +* Data: the complete desired agent credential collection. + +After creating it, set: + +```yaml +spec: + credentialsRef: + name: kars-credential-source-demo + uid: +``` + +There is no namespace field in the reference. The controller rejects arbitrary +source names, missing purpose/intent, different target/workspace, different UID, +non-Opaque type, foreign owner references, or stale bindings. Initial binding +records the exact Sandbox UID and runtime namespace UID, with a UID/RV-fenced +metadata-only patch. Existing binding metadata must agree; it is not overwritten +to claim another Sandbox's source. + +An intentionally replaced source requires a new explicit reference. +`credentials update --use-source` selects a new **compatible** source incarnation; +ordinary updates refuse UID drift. Foreign purpose/ownership is never adopted, +even with `--use-source`. + +## Collection semantics and supported keys + +While a reference is present, **only the owned projection is mounted as the +agent credential collection**. The direct Secret is not a fallback. Therefore +removing a source key cannot resurrect an old same-named legacy key. + +Version 1 uses the existing handoff channel/search credential allowlist: + +`TELEGRAM_BOT_TOKEN`, `TELEGRAM_ALLOW_FROM`, `SLACK_BOT_TOKEN`, +`DISCORD_BOT_TOKEN`, `WHATSAPP_ENABLED`, `BRAVE_API_KEY`, `TAVILY_API_KEY`, +`EXA_API_KEY`, `FIRECRAWL_API_KEY`, `PERPLEXITY_API_KEY`. + +Values must be UTF-8 without NUL; the collection is limited to 128 KiB. +Router/provider/control-plane and arbitrary process environment keys are not +accepted. In particular, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, Azure identity +keys, `AGT_*`, `KARS_*`, `NODE_OPTIONS`, and `PATH` cannot be supplied this way. +Existing direct-mode flags, including `--openai-api-key`, remain available. +Migration fails rather than silently dropping unsupported existing keys. + +Credential-key overrides in runtime `extraEnv`/raw `env` conflict with source +mode and are rejected, rather than silently overriding the source. The common +agent container path supports every **wired, controller-managed** runtime, +including BYO. The projection is never added to the inference-router EnvFrom. +Overlay-managed pods and unwired runtime variants are not supported. + +## Projection, refresh, and failure handling + +The controller owns `-credential-projection` in the verified +runtime namespace. Its metadata binds purpose, workspace, Sandbox UID, namespace +UID, and source UID, with a namespace owner reference. Unrelated Secrets at that +name are not overwritten or adopted. + +An empty metadata-only anchor is created first and sealed to its own Secret UID. +Recreating an object with copied old projection metadata does not authorize +adoption of the new UID. The controller rechecks the +Sandbox UID/RV, namespace incarnation, and source UID/RV before a UID/RV-fenced +value patch. Removed keys use explicit nulls. No force-apply is used for the +projection. Credential values and API bodies are excluded from errors/logs. + +Source-owned Secrets are watched through kube's **metadata-only ownership +watch**. A 30-second reconciliation backstop covers deletion, removed ownership +markers, unbound/invalid sources, and projection drift. Projection UID/RV drives +the pod-template revision; metadata-only source changes do not restart pods. + +Source mode uses `Recreate`, not a rolling update that could indefinitely retain +an old credential-bearing pod when a replacement fails. Changes pause the +verified controller Deployment before refresh. Invalid, missing, or replaced +references stop that runtime and clear only the owned projection, then report +`CredentialSourceUnavailable` and retry. They never fall back to direct +credentials. Concurrent writes/replacements and non-404 API failures propagate +honestly; retries restart with fresh identities. + +Kubernetes is not a cross-resource transaction system. Watches/requeues make +revocation asynchronous; API outages, unreachable nodes, and pod termination +failures can delay it. This feature does not revoke credentials at their external +provider, erase values previously observed by an agent, or defend against a +cluster administrator deliberately bypassing namespace lifecycle controls. + +## Bridge and publication boundary + +This is a standalone core primitive, **not closure of Bridge RBAC**. A later +adapter must store the source in its configured workspace and include the exact +UID in launch bindings. Existing Bridge `put_credential` behavior is unchanged. +Workspace Secret permissions/admission must still protect signing and other +control-plane Secrets; no new broad Role or binding is introduced here. diff --git a/docs/how-to/helm-installation.md b/docs/how-to/helm-installation.md index 70ba0ea2..f72c1e6c 100644 --- a/docs/how-to/helm-installation.md +++ b/docs/how-to/helm-installation.md @@ -9,6 +9,10 @@ updated CLI and resolve all ownership conflicts. See [namespace ownership migration and adoption](namespace-ownership.md). The check does not read Secrets or change namespaces, workloads, or Helm values. +Optional [workspace credential sources](credential-sources.md) require the +matching controller and CRD. Upgrade both before using `--credential-source`; +legacy direct credential Secrets remain the default. + ## Local kind ```bash diff --git a/docs/security-audits/2026-09-07-credential-sources.md b/docs/security-audits/2026-09-07-credential-sources.md new file mode 100644 index 00000000..0241faae --- /dev/null +++ b/docs/security-audits/2026-09-07-credential-sources.md @@ -0,0 +1,166 @@ +# Agent credential-source capability review — 2026-09-07 + +**Status:** additive candidate with maintainer sign-off received; independent +human security review and sign-off remain pending. Automated tests are evidence, +not another person's approval. No organizational approval, compliance +certification, or production qualification is asserted by this document. + +## Scope and base + +Base: namespace prerequisite `62093414cb8d5d9937c1d9974504047c84669d6c` +(Azure/kars#548). This slice adds an optional UID-pinned KarsSandbox credential +source, controller-owned projection, CLI source lifecycle, and schema/docs. +It does not change Bridge permissions, inference credentials, signing providers, +agent transport, or customer/live deployments. + +The optional field is omitted from serialization when unset, preserving old +Sandbox-spec inputs. When present, its name/UID remain in serialized spec +digests. Existing Task/Team blueprint and receipt formats are not extended: +mutable credential values are not claimed to be covered by a governance receipt. + +## Authority boundaries + +| Boundary | Control | +|---|---| +| Arbitrary Secret read oracle | Fixed source-name derivation; same CR workspace; exact UID; explicit purpose/intent/target before reading values | +| Source type and data | Opaque only; existing agent channel/search allowlist; provider/control-plane/process keys rejected | +| Sandbox recreation | Exact source owner binding to Sandbox UID; no automatic name-only future delivery | +| Workspace/name collisions | Namespace claim v1 and runtime namespace UID rechecks | +| Foreign projection | Exact projection purpose, namespace owner reference, Sandbox/workspace/namespace UID bindings | +| Cross-resource write races | Empty anchor, source/Sandbox/namespace rechecks, UID/RV-fenced final value patch; no force takeover | +| Rotation/removal | Exclusive collection, explicit removed-key nulls, UID/RV revision, controlled stop/Recreate | +| Source invalidation | Stop verified runtime, revoke only owned projection, report failure, bounded retry; no legacy fallback | +| Error disclosure | Kubernetes API errors reduced to stage/status; no request bodies or credential values in source errors | +| CLI authority | Source before CR CREATE; exact returned UID; source update CAS; explicit UID reselection; schema-retention and controller-version acknowledgement checks | + +Unconfigured Sandboxes keep their direct EnvFrom collection. Explicit opt-in +performs a one-time CLI migration without editing the direct Secret. Explicit +opt-out restores that direct collection. Neither operation may silently move +foreign or provider/control-plane data through the source primitive. + +## Automated evidence + +Existing Rust/Vitest/Helm runners cover: + +* Missing-reference legacy behavior and default schema/serialization. +* Explicit source creation/binding and fenced projection writes. +* Key addition, update, removal, empty collections, and metadata-only changes. +* Missing/replaced source, wrong workspace/name, foreign owner/type/purpose, + namespace replacement, destination collision, and API/CAS races. +* Owned-only revocation and controller deployment pause/refresh. +* Source values excluded from error text and process-environment injection. +* CLI pre-CR storage, actual returned UID binding, collection migration, + update/removal/disable, safe errors, and direct-path compatibility. +* Generated/reference schema parity and Helm admission-rule rendering. + +Candidate qualification on 2026-09-07: + +* 66 targeted controller tests passed (credential-source, namespace ownership, + and SRE-writer compatibility selectors), offline/locked with incremental + compilation disabled. +* Controller all-targets Clippy passed with warnings denied. +* 90 targeted CLI/Vitest tests passed, including Helm rendering and new-module + size/header/no-stub/no-custom-crypto assertions. +* TypeScript typecheck and changed-file oxlint passed. + +The existing disposable Kind harness now includes a source-bound BYO fixture +using its already-loaded sandbox test image. The consumer emits fixed markers, +not environment values. The lifecycle covers actual process environment on +initial delivery, rotation and key removal; missing-source revocation with no +legacy fallback; explicit opt-out restoring the preserved direct collection; +and cleanup preserving the core namespace. Shell syntax is checked locally; +execution of this new case awaits hosted CI. + +Hosted Kind execution at `b69a6ad6` exposed a real installation blocker: +the new CEL rule referenced `upstreamCompatibility`, which was missing from the +handwritten Helm schema. Kubernetes rejected the entire Sandbox CRD, so no +credential-consumer lifecycle result was established. Local rendering and +source review were insufficient to catch that API-server compilation failure. + +The repair declares the existing Rust compatibility fields in the optional +Helm schema, retains the source/overlay admission guard, and requires an upstream +reference for overlay mode. The targeted schema assertion now checks the +referenced field definitions, not just the presence of rule text. The Kind +fixture also requires the intended overlay rejection message, and Helm setup +failure now stops the harness instead of producing cascading secondary failures. +Fifteen targeted schema/Helm tests, typecheck, scoped lint, Helm lint and shell +syntax pass locally; the repaired head still requires real API-server execution. + +The next hosted run at `693a46ca` installed the CRD and passed the intended +overlay admission rejection. The real BYO consumer then passed initial +delivery, rotation with key removal, source-deletion revocation without legacy +fallback, and explicit opt-out. Sandbox namespace cleanup also completed. +The run subsequently hit its job deadline while deleting the InferencePolicy +fixture; full lifecycle cleanup is therefore not qualified. The fixture now +bounds that deletion to 90 seconds and captures policy/ConfigMap/controller +diagnostics on failure rather than hanging until the job cancels. This is a +diagnostic change, not a claimed repair of the underlying cleanup failure. + +The bounded follow-up at `d444656b` identified the cleanup cause: the +InferencePolicy reconciler sent a partial server-side-apply finalizer payload +without `metadata.name`, which the API server rejected with HTTP 400 after the +profile ConfigMap had already been deleted. The same error affected SRE's +inference policy. The repair uses complete name/namespace/UID/resourceVersion +metadata in merge patches for finalizer registration and removal, preserving +unrelated finalizers. Profile cleanup errors other than 404 now propagate before +the cleanup finalizer can be released. No force apply or finalizer bypass is +used. Fourteen focused reconciler tests, including HTTP request shape, +cleanup ordering and API/CAS error cases, and strict controller all-target +Clippy pass. Real Kind cleanup and independent repair review remain pending. + +Diff-based publication gates still run on the parent's eventual atomic commit; +the candidate was intentionally not committed or pushed here. No live +Kubernetes admission or customer rollout was executed. Real-cluster +qualification and independent/human review remain release gates. + +## Remaining limits and required review + +1. Kubernetes source/target operations are not transactional. Last-read identity + checks plus a fenced target write prevent blind replacement writes; watches + and a bounded retry provide eventual source revocation. +2. API failures/node partitions can delay process termination. External + credential invalidation and secrets already observed by an agent are outside + this feature's guarantees. +3. Namespace administrators remain trusted. Namespace claim annotations are not + cryptographic attestations or protection from forced namespace finalization. +4. CLI credential flags retain their existing shell/process-argument exposure. + Kubernetes source payloads themselves travel through stdin and are not logged. +5. Bridge still needs a separate reviewed adapter and least-privilege/admission + changes. This candidate must not be described as Bridge Secret-RBAC closure. +6. Review schema/controller mixed-version handling, consumer rollout behavior, + and source-authoring permissions before release. + +## Qualification and maintainer sign-off update + +The earlier pending technical evidence above is superseded by closure at +`8b206065608593667a40665b3f48225ef9ce278d` in Azure/kars#549. Automated bounded +review closed the finalizer repair, and all hosted technical gates passed. +The real disposable Kind lifecycle completed 111 cases with zero failures, +including consumer rotation/revocation, explicit opt-out, mandatory policy +cleanup and preservation of the core namespace: +https://github.com/Azure/kars/actions/runs/34169168508/job/101889685225. +This is not a live customer deployment or Bridge credential/RBAC closure. + +The maintainer explicitly signed off and approved that qualified source on +2026-09-08. This does not approve later functional changes, other unresolved +slices, customer deployment or promotion to `main`. The separate independent +reviewer must provide their own sign-off before the two-person audit gate can +pass; this single maintainer approval must not be counted twice. + +Signed-off-by: pallakatos <191481949+pallakatos@users.noreply.github.com> + +## Explicit author waiver for integration assembly + +On 2026-09-08, Kars author `pallakatos` explicitly waived the second-person +sign-off for Azure/kars#549: "just push them and say I waived it". +This supersedes the independent-signature landing requirement above only for +assembling the already-qualified source into `Azure/kars:kars-bridge`. +It does not assert that an independent human review occurred. + +All other required technical and security gates must pass on the landing head. +The signature-only branch-protection exception and the already-authorized +account-specific review allowance must be restored immediately after the merge, +including on failure. No CI result is rewritten as successful. +The waiver does not cover functional changes beyond the qualified source, +later unresolved slices, `main` promotion, customer deployments or public +publication of the private Bridge application. diff --git a/tests/e2e/credential-sources.sh b/tests/e2e/credential-sources.sh new file mode 100644 index 00000000..0618b015 --- /dev/null +++ b/tests/e2e/credential-sources.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Uses only the existing disposable Kind cluster and its loaded BYO test image. +# The consumer emits fixed markers, never environment values. +cleanup_credential_source_policy() { + local k=(kubectl --context kind-kars-e2e) + if ! "${k[@]}" delete inferencepolicy e2e-source-inference -n kars-system \ + --timeout=90s --request-timeout=20s >/dev/null; then + fail "Credential-source InferencePolicy cleanup did not complete within 90s" + "${k[@]}" get inferencepolicy e2e-source-inference -n kars-system \ + --request-timeout=20s -o yaml || true + "${k[@]}" get configmap inferencepolicy-e2e-source-inference-profile -n kars-system \ + --request-timeout=20s -o go-template='{{.metadata}}{{"\n"}}' || true + "${k[@]}" logs -n kars-system -l app.kubernetes.io/component=controller \ + --tail=1000 --since=5m --request-timeout=20s || true + return 1 + fi +} + +wait_for_credential_consumer() { + local marker="$1" deadline=$(($(date +%s) + 150)) pod namespace + local k=(kubectl --context kind-kars-e2e) + while [ "$(date +%s)" -lt "$deadline" ]; do + namespace=$("${k[@]}" get namespace kars-e2e-source --ignore-not-found \ + -o jsonpath='{.metadata.uid}') || return 1 + if [ -z "$namespace" ]; then + sleep 2 + continue + fi + pod=$("${k[@]}" get pods -n kars-e2e-source -l kars.azure.com/sandbox=e2e-source \ + -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' \ + 2>/dev/null | head -1) || return 1 + if [ -n "$pod" ] && "${k[@]}" logs "$pod" -n kars-e2e-source -c agent --tail=10 2>/dev/null \ + | grep -qx "$marker"; then + return 0 + fi + sleep 2 + done + fail "Credential consumer did not reach $marker" + return 1 +} + +test_credential_sources() { + local k=(kubectl --context kind-kars-e2e) + local source_uid projection_uid direct_uid namespace_uid projection admission + namespace_uid=$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}') || return 1 + if admission=$("${k[@]}" create --dry-run=server -f - 2>&1 <<'YAML' +apiVersion: kars.azure.com/v1alpha1 +kind: KarsSandbox +metadata: + name: e2e-source-overlay + namespace: kars-system +spec: + inferenceRef: + name: e2e-source-inference + runtime: + kind: BYO + byo: + image: kars-sandbox-e2e:dev + contractVersion: v1 + sandbox: + isolation: standard + credentialsRef: + name: kars-credential-source-e2e-source-overlay + uid: fixture-uid + upstreamCompatibility: + sigsAgentSandbox: overlay + upstreamSandboxRef: + name: upstream-fixture +YAML + ); then + fail "Admission accepted source credentials on an overlay-managed runtime"; return 1 + fi + if ! printf '%s\n' "$admission" | grep -q "credentialsRef requires a controller-managed runtime"; then + printf '%s\n' "$admission" + fail "Overlay admission failed for a reason other than the intended source guard"; return 1 + fi + pass "The API server compiles the source schema and rejects overlay credentials for the intended reason" + if ! "${k[@]}" create -f - <<'YAML' +apiVersion: v1 +kind: Secret +metadata: + name: kars-credential-source-e2e-source + namespace: kars-system + annotations: + kars.azure.com/credential-purpose: agent-source-v1 + kars.azure.com/credential-target: e2e-source + kars.azure.com/credential-workspace: kars-system + kars.azure.com/credential-binding-intent: explicit-reference-v1 +type: Opaque +stringData: + BRAVE_API_KEY: fixture-one + TAVILY_API_KEY: fixture-auxiliary +--- +apiVersion: kars.azure.com/v1alpha1 +kind: InferencePolicy +metadata: + name: e2e-source-inference + namespace: kars-system +spec: + appliesTo: + sandboxName: e2e-source + modelPreference: + primary: + provider: azure-openai + deployment: gpt-4.1 +YAML + then + fail "Cannot create the credential-source fixtures"; return 1 + fi + source_uid=$("${k[@]}" get secret kars-credential-source-e2e-source -n kars-system \ + -o jsonpath='{.metadata.uid}') || return 1 + [ -n "$source_uid" ] || { fail "Source fixture omitted its UID"; return 1; } + if ! "${k[@]}" create -f - </dev/null || return 1 + direct_uid=$("${k[@]}" get secret e2e-source-credentials -n kars-e2e-source \ + -o jsonpath='{.metadata.uid}') || return 1 + "${k[@]}" patch secret kars-credential-source-e2e-source -n kars-system --type=merge \ + -p '{"stringData":{"BRAVE_API_KEY":"fixture-two"},"data":{"TAVILY_API_KEY":null}}' \ + >/dev/null || return 1 + wait_for_credential_consumer CREDENTIAL_FIXTURE_ROTATED || return 1 + if [ "$("${k[@]}" get secret e2e-source-credential-projection -n kars-e2e-source -o jsonpath='{.metadata.uid}')" != "$projection_uid" ]; then + fail "Rotation replaced the projection instead of updating the owned Secret"; return 1 + fi + pass "Source rotation replaces the consumer and removed keys do not fall back to legacy values" + + "${k[@]}" delete secret kars-credential-source-e2e-source -n kars-system --timeout=30s >/dev/null || return 1 + local deadline=$(($(date +%s) + 150)) revoked=0 reason replicas pods keys + while [ "$(date +%s)" -lt "$deadline" ]; do + reason=$("${k[@]}" get karssandbox e2e-source -n kars-system \ + -o jsonpath='{.status.conditions[?(@.type=="Degraded")].reason}') || return 1 + replicas=$("${k[@]}" get deployment e2e-source -n kars-e2e-source \ + -o jsonpath='{.spec.replicas}') || return 1 + pods=$("${k[@]}" get pods -n kars-e2e-source -l kars.azure.com/sandbox=e2e-source \ + -o go-template='{{len .items}}') || return 1 + keys=$("${k[@]}" get secret e2e-source-credential-projection -n kars-e2e-source \ + --ignore-not-found -o go-template='{{if .data}}{{len .data}}{{else}}0{{end}}') || return 1 + if [ "$reason" = CredentialSourceUnavailable ] && [ "$replicas" = 0 ] && [ "$pods" = 0 ] \ + && { [ -z "$keys" ] || [ "$keys" = 0 ]; }; then + revoked=1 + break + fi + sleep 2 + done + if [ "$revoked" != 1 ]; then + fail "Source deletion did not stop the consumer and revoke its owned projection"; return 1 + fi + if [ "$("${k[@]}" get secret e2e-source-credentials -n kars-e2e-source -o jsonpath='{.metadata.uid}')" != "$direct_uid" ]; then + fail "Source revocation changed the unrelated legacy Secret"; return 1 + fi + pass "Source deletion revokes the consumer without reactivating or deleting legacy credentials" + + "${k[@]}" patch karssandbox e2e-source -n kars-system --type=merge \ + -p '{"spec":{"credentialsRef":null}}' >/dev/null || return 1 + wait_for_credential_consumer CREDENTIAL_FIXTURE_LEGACY || return 1 + projection=$("${k[@]}" get secret e2e-source-credential-projection -n kars-e2e-source \ + --ignore-not-found -o name) || return 1 + if [ -n "$projection" ]; then + fail "Explicit source opt-out left its owned projection behind"; return 1 + fi + pass "Explicit opt-out restores the unchanged legacy collection only when requested" + + "${k[@]}" delete karssandbox e2e-source -n kars-system --wait=false >/dev/null || return 1 + "${k[@]}" wait --for=delete namespace/kars-e2e-source --timeout=120s || return 1 + cleanup_credential_source_policy || return 1 + if [ "$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}')" != "$namespace_uid" ]; then + fail "Credential-source cleanup changed the core namespace"; return 1 + fi + pass "Credential-source lifecycle cleanup preserves the core namespace" +} diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 34536369..9e9c58cc 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -188,6 +188,7 @@ install_crds() { kubectl get all -n kars-system || true kubectl describe pod -n kars-system -l app.kubernetes.io/component=controller || true kubectl logs -n kars-system -l app.kubernetes.io/component=controller --tail=200 || true + return 1 fi } @@ -3015,6 +3016,7 @@ EOF # ─── Main ───────────────────────────────────────────────────────────────────── source "$SCRIPT_DIR/namespace-ownership.sh" +source "$SCRIPT_DIR/credential-sources.sh" main() { echo "" @@ -3115,6 +3117,7 @@ main() { esac test_sre_namespace_ownership || fail "SRE namespace lifecycle gate failed" + test_credential_sources || fail "Credential-source lifecycle gate failed" echo "" echo "═══════════════════════════════════════════════════════"