diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 716b086df..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,9 @@ jobs: # transitive oci-client→reqwest) compiles natively with the # runner's gcc. run: cargo build --release --workspace + - name: Legacy Hermes HTTPS client test dependency + if: needs.changes.outputs.code == 'true' + run: python3 -m pip install 'httpx==0.28.1' - name: cargo nextest run (release) if: needs.changes.outputs.code == 'true' # Reuses the target/release/ artefacts the previous step just @@ -393,6 +396,67 @@ jobs: - name: RBAC idempotency gate run: python3 ci/bicep-rbac-idempotency.py + sre-crd-schema: + name: SRE CRD API Schema + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + KUBECONFIG: ${{ github.workspace }}/.e2e-sre-schema-kubeconfig + PYTHONDONTWRITEBYTECODE: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Same pinned tools and unmodified node configuration as e2e-kind. + # Kind v0.24.0 defaults to the observed kindest/node:v1.31.0 image. + - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 + with: + install_only: true + version: v0.24.0 + - uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4 + with: + version: v1.30.5 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Check public-schema diagnostic privacy + run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test + - name: Create the same disposable API server as the real harness + run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" + - name: Prove native historical Helm wait orders CRDs before hooks and permits schema upgrades + run: PYTHONPATH=tests/e2e python3 -m sre_authority.legacy_crd_probe + - name: Reset disposable cluster after historical Helm proof + run: | + kind delete cluster --name kars-e2e + kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" + - name: Validate the SRE CRD against the actual API server + id: sre_schema + run: python3 tests/e2e/sre_authority/registration_schema.py --exercise + - name: Prove controller Pod admission with all chart policies and no image execution + id: sre_bootstrap + run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --retirement-bind-proof + - name: Collect nil-schema adapter candidate evidence without weakening production gates + if: failure() && steps.sre_bootstrap.outcome == 'failure' + run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --json-params-candidate + - name: Collect namespace-accessor candidate evidence without relaxing the failing production gate + if: failure() && steps.sre_schema.outcome == 'failure' + run: python3 tests/e2e/sre_authority/registration_schema.py --namespace-accessor-candidate --exercise + - name: Upload only public CRD schema evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 + with: + name: sre-crd-schema-${{ github.run_id }} + path: | + e2e-sre-schema-diag/versions.json + e2e-sre-schema-diag/legacy-helm-readiness.json + e2e-sre-schema-diag/validation.json + e2e-sre-schema-diag/validation-instances.json + e2e-sre-schema-diag/namespace-accessor-candidate.json + e2e-sre-schema-diag/namespace-accessor-candidate-instances.json + e2e-sre-schema-diag/bootstrap-*.json + e2e-sre-schema-diag/candidate-bootstrap-*.json + if-no-files-found: warn + retention-days: 7 + - name: Remove disposable schema cluster + if: always() + run: kind delete cluster --name kars-e2e + helm-lint: name: Helm Lint runs-on: ubuntu-latest @@ -572,7 +636,7 @@ jobs: # Fetch enough history to diff. git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true if git diff --name-only "$base" "$head" 2>/dev/null \ - | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then + | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|shared/|runtimes/hermes/src/kars_runtime_hermes/plugin/sre|cli/src/(commands/sre|lib/sre|lib/namespace-ownership)|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" @@ -616,6 +680,21 @@ jobs: docker system prune -af --volumes >/dev/null 2>&1 || true df -h + - name: Set up Node for real SRE authority CLI acceptance + if: steps.paths.outputs.run == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + cache: npm + cache-dependency-path: cli/package-lock.json + - name: Build locked CLI for Kind acceptance + if: steps.paths.outputs.run == 'true' + working-directory: cli + run: npm ci && npm run build + - name: SRE Kind unchanged Hermes HTTPS client dependency + if: steps.paths.outputs.run == 'true' + run: python3 -m pip install 'httpx==0.28.1' + # Pre-built binaries from build-rust are COPY'd into the # distroless runtime images; no `cargo build` runs inside Docker. # Result: each image build is ~30s instead of ~8min. @@ -690,7 +769,12 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 with: name: e2e-diagnostics-${{ github.run_id }} - path: e2e-diag/ + path: | + e2e-diag/ + e2e-sre-schema-diag/versions.json + e2e-sre-schema-diag/validation.json + e2e-sre-schema-diag/registration-create.json + e2e-sre-schema-diag/bootstrap-*.json retention-days: 7 bench-regression: diff --git a/Cargo.lock b/Cargo.lock index 44e7cfddf..c9a7577b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2501,6 +2501,7 @@ dependencies = [ "oci-client 0.16.1", "prometheus", "rand 0.9.4", + "rcgen", "regex", "reqwest 0.12.28", "rustls", @@ -2511,6 +2512,7 @@ dependencies = [ "sha2 0.10.9", "sigstore", "thiserror 2.0.18", + "time", "tokio", "tokio-tungstenite 0.28.0", "tracing", @@ -2552,8 +2554,10 @@ dependencies = [ "prometheus", "proptest", "rand 0.9.4", + "rcgen", "reqwest 0.12.28", "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_yaml", @@ -2562,6 +2566,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls", "tokio-stream", "tokio-tungstenite 0.26.2", "tokio-util", @@ -4034,6 +4039,19 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -6343,6 +6361,15 @@ dependencies = [ "tls_codec", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 6e451a538..5e3ef964b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,10 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" # with "Could not automatically determine the process-level # CryptoProvider" when multiple feature flags resolve. rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs"] } +rcgen = { version = "0.13.2", default-features = false, features = ["aws_lc_rs", "pem"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs"] } +rustls-pemfile = "2" +time = "0.3" tower = { version = "0.5", features = ["limit"] } tower-http = { version = "0.6", features = ["trace", "cors"] } hyper = "1" diff --git a/ci/no-custom-crypto.sh b/ci/no-custom-crypto.sh index 8e282611c..e9f1f8c13 100755 --- a/ci/no-custom-crypto.sh +++ b/ci/no-custom-crypto.sh @@ -67,6 +67,7 @@ ALLOW_PATHS=( # Production paths to scan. PROD_PATHS=( + 'shared/' 'controller/src/' 'inference-router/src/' 'cli/src/' diff --git a/ci/no-stubs.sh b/ci/no-stubs.sh index ae5d8beb4..12839ca3d 100755 --- a/ci/no-stubs.sh +++ b/ci/no-stubs.sh @@ -19,6 +19,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" PROD_PATHS=( + 'shared/' 'controller/src/' 'inference-router/src/' 'cli/src/' diff --git a/ci/security-audit-required.sh b/ci/security-audit-required.sh index b9c37bc25..e2aa54480 100755 --- a/ci/security-audit-required.sh +++ b/ci/security-audit-required.sh @@ -17,7 +17,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" # Capability-introducing paths — mirrors §4.4 of the plan. -CAP_RE='^(controller/src/(crd|reconcilers|admission)|inference-router/src/(mcp|a2a|providers|routes)|cli/src/(commands|migrate|adapters)|runtimes/openclaw/src/(core|index\.ts)|sandbox-images/[^/]+/(Dockerfile|entrypoint\.sh)|cli/profiles/|deploy/seccomp/|deploy/helm/kars/files/)' +CAP_RE='^(controller/src/(crd|reconcilers|admission)|inference-router/src/(mcp|a2a|providers|routes)|cli/src/(commands|migrate|adapters)|runtimes/openclaw/src/(core|index\.ts)|sandbox-images/[^/]+/(Dockerfile|entrypoint\.sh)|cli/profiles/|deploy/seccomp/|deploy/helm/kars/files/|shared/.*\.rs$)' changed=$(git diff --name-only "${BASE_REF}...HEAD" 2>/dev/null || git diff --name-only HEAD) # Exclude test files — they exercise capabilities but don't introduce diff --git a/cli/src/commands/destroy.test.ts b/cli/src/commands/destroy.test.ts new file mode 100644 index 000000000..67e698af1 --- /dev/null +++ b/cli/src/commands/destroy.test.ts @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { destroyCommand } from "./destroy.js"; + +const { execute, target, guard } = vi.hoisted(() => ({ + execute: vi.fn(), target: vi.fn(), guard: vi.fn(), +})); +vi.mock("execa", () => ({ execa: execute })); +vi.mock("../lib/azure-destroy-target.js", () => ({ withAzureDestroyTarget: target })); +vi.mock("../lib/sre-authority.js", () => ({ assertDestroySafe: guard })); +vi.mock("ora", () => ({ default: () => ({ start() { return this; }, succeed: vi.fn(), fail: vi.fn() }) })); +afterEach(() => { vi.restoreAllMocks(); vi.clearAllMocks(); }); + +describe("destroy --all cannot bypass target-bound retirement", () => { + it.each([["--local"], ["--local", "--cloud"], ["sre", "--local"], ["anything", "--local"]].map(flags => ({ flags })))( + "rejects Azure --all with local flags: $flags", async ({ flags }) => { + await expect(destroyCommand().parseAsync(["node", "destroy", "--all", "--yes", ...flags])).rejects.toThrow("cannot be combined"); + expect(execute).not.toHaveBeenCalled(); + expect(target).not.toHaveBeenCalled(); + }, + ); + it.each([[], ["sre"], ["unrelated"], ["--cloud"]].map(flags => ({ flags })))("always binds Azure deletion for $flags", async ({ flags }) => { + const azure = vi.fn().mockResolvedValue({ stdout: "" }); + target.mockImplementation(async (_exec, _rg, _sub, _context, destroy) => destroy(azure)); + await destroyCommand().parseAsync(["node", "destroy", "--all", "--yes", "--resource-group", "B", + "--subscription", "subscription-id", "--context", "A", ...flags]); + expect(target).toHaveBeenCalledWith(execute, "B", "subscription-id", "A", expect.any(Function)); + expect(guard).not.toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + expect(azure.mock.calls.map(([, args]) => args.slice(0, 2))).toEqual([ + ["group", "delete"], ["cognitiveservices", "account"], ["keyvault", "purge"], + ]); + }); + it("never starts deletion when target proof fails", async () => { + target.mockRejectedValue(new Error("target mismatch")); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("exit 1"); }); + await expect(destroyCommand().parseAsync(["node", "destroy", "--all", "--yes"])).rejects.toThrow("exit 1"); + expect(execute).not.toHaveBeenCalled(); + }); + it("does not fetch credentials or mutate anything merely to show --all confirmation", async () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + await destroyCommand().parseAsync(["node", "destroy", "--all"]); + expect(target).not.toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + }); +}); diff --git a/cli/src/commands/destroy.ts b/cli/src/commands/destroy.ts index 41897be4c..decf6420b 100644 --- a/cli/src/commands/destroy.ts +++ b/cli/src/commands/destroy.ts @@ -4,6 +4,8 @@ import { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; +import { assertDestroySafe } from "../lib/sre-authority.js"; +import { withAzureDestroyTarget } from "../lib/azure-destroy-target.js"; export function destroyCommand(): Command { const cmd = new Command("destroy"); @@ -16,12 +18,19 @@ export function destroyCommand(): Command { .option("--cloud", "Destroy AKS cloud sandbox only (skip Docker)", false) .option("--all", "Destroy ALL resources (AKS, ACR, KV, AOAI — deletes the resource group)", false) .option("-g, --resource-group ", "Resource group name") + .option("--subscription ", "Azure subscription for --all (defaults to the selected Azure account)") .option("--region ", "Azure region (used to derive resource group)", "eastus2") - .option("--context ", "Kubernetes context to use (defaults to current)") + .option("--context ", "Kubernetes context (current for sandbox teardown; verified against Azure target for --all)") .action(async (name: string | undefined, options) => { const rg = options.resourceGroup || `kars-${options.region}`; // Propagate --context to every kubectl invocation in this command. const kctlCtx = options.context ? ["--context", options.context] : []; + if (options.all && options.local) throw new Error("--all deletes Azure resources and cannot be combined with --local"); + if (!options.all && (!options.local || options.cloud) && (!name || name === "sre")) { + const { execa } = await import("execa"); + await assertDestroySafe((file,args,commandOptions) => + execa(file,[...kctlCtx,...args],commandOptions)); + } if (options.all) { // Full teardown — delete the entire resource group @@ -41,25 +50,27 @@ export function destroyCommand(): Command { const { execa } = await import("execa"); const baseName = "kars"; - // Delete the resource group (async) - await execa("az", [ - "group", "delete", "--name", rg, "--yes", "--no-wait", "--output", "none", - ], { stdio: "pipe" }); - - // Purge soft-deleted resources so a fresh 'up' works without conflicts - spinner.text = "Purging soft-deleted Azure OpenAI account..."; - await execa("az", [ - "cognitiveservices", "account", "purge", - "--name", `${baseName}-aoai`, - "--resource-group", rg, - "--location", options.region, - "--output", "none", - ], { stdio: "pipe" }).catch(() => {}); + await withAzureDestroyTarget(execa, rg, options.subscription, options.context, async azure => { + // Delete the resource group (async) + await azure("az", [ + "group", "delete", "--name", rg, "--yes", "--no-wait", "--output", "none", + ], { stdio: "pipe" }); - spinner.text = "Purging soft-deleted Key Vault..."; - await execa("az", [ - "keyvault", "purge", "--name", `${baseName}-kv`, - ], { stdio: "pipe" }).catch(() => {}); + // Purge soft-deleted resources so a fresh 'up' works without conflicts + spinner.text = "Purging soft-deleted Azure OpenAI account..."; + await azure("az", [ + "cognitiveservices", "account", "purge", + "--name", `${baseName}-aoai`, + "--resource-group", rg, + "--location", options.region, + "--output", "none", + ], { stdio: "pipe" }).catch(() => {}); + + spinner.text = "Purging soft-deleted Key Vault..."; + await azure("az", [ + "keyvault", "purge", "--name", `${baseName}-kv`, + ], { stdio: "pipe" }).catch(() => {}); + }); spinner.succeed(`Resource group '${rg}' deletion initiated + soft-deleted resources purged`); } catch (error) { diff --git a/cli/src/commands/dev/local-k8s.ts b/cli/src/commands/dev/local-k8s.ts index c3aadd388..32f7f921b 100644 --- a/cli/src/commands/dev/local-k8s.ts +++ b/cli/src/commands/dev/local-k8s.ts @@ -31,6 +31,7 @@ import { ensureAgtRepo, ensureAgtWheels } from "../../lib/agt-bootstrap.js"; import { resolveBundledAsset, requireBundledAsset, findRepoRootOrNull } from "../../lib/repo-assets.js"; import { buildCopilotFallbackChain } from "../../github-copilot.js"; import { CLAIM, prepareCredentialNamespace } from "../../lib/namespace-ownership.js"; +import { assertSafeMutation } from "../../lib/sre-authority.js"; export interface LocalK8sOptions { /** Sandbox / agent name. Reused as Helm release name suffix. */ @@ -1337,6 +1338,10 @@ export async function runLocalK8s(opts: LocalK8sOptions): Promise { stepper.step(`Ensuring kind cluster '${opts.clusterName}' exists…`); await ensureCluster(tools.kind, opts.clusterName, tools.env); + await assertSafeMutation((file,args,commandOptions) => execa( + file === "kubectl" ? tools.kubectl : file, + ["--context", `kind-${opts.clusterName}`, ...args], commandOptions, + )); stepper.done(`kind cluster '${opts.clusterName}' is ready`); // Ensure the three local-dev images exist AND match the host arch. diff --git a/cli/src/commands/push-apply.test.ts b/cli/src/commands/push-apply.test.ts index 5b4388bc5..079c23e37 100644 --- a/cli/src/commands/push-apply.test.ts +++ b/cli/src/commands/push-apply.test.ts @@ -146,6 +146,14 @@ function fixture(options: { legacyCore?: boolean; mesh?: "absent" | "helm" | "ex } describe("selected core push artifacts", () => { + it("still requires SRE authority preflight before resolving or applying valid core artifacts", async () => { + const f = fixture({ fail: "karssreregistrations.kars.azure.com" }); + await expect(applyPushedImages(f.execute, [pushed("controller")], "chart")) + .rejects.toThrow("karssreregistrations.kars.azure.com"); + expect(f.calls().some(([bin, args]) => bin === "az" + || ["upgrade", "patch", "annotate", "rollout"].includes(args[0]))).toBe(false); + }); + it("moves a GHCR/pinned Helm controller to its pushed ACR digest without resetting customer values", async () => { const f = fixture(); await applyPushedImages(f.execute, [pushed("controller")], "chart"); diff --git a/cli/src/commands/push-apply.ts b/cli/src/commands/push-apply.ts index ff47f1405..81202b82c 100644 --- a/cli/src/commands/push-apply.ts +++ b/cli/src/commands/push-apply.ts @@ -11,6 +11,7 @@ import { import { coreImageValues, imageValueArgs, PUSH_COMPONENTS, resolvePushedArtifacts, type PushedImage } from "../lib/image-targets.js"; import { inspectCoreInstallation, recheckCoreOwnership, requireHealthyDeployment, updateLegacyCore, verifyCoreConfiguration } from "../lib/core-image-apply.js"; import { inspectSandboxPlans, refreshSandboxImages } from "../lib/sandbox-image-apply.js"; +import { assertSafeMutation } from "../lib/sre-authority.js"; export interface PushApplyResult { applied: string[]; @@ -41,6 +42,7 @@ export async function applyPushedImages( throw new Error("External or absent AgentMesh cannot be updated; choose explicit core targets instead."); } if (core?.kind === "helm" && mesh) assertMeshReleaseConsistency(mesh, core.values); + await assertSafeMutation(execute); const artifacts = await resolvePushedArtifacts(execute, deployable); const coreImages = artifacts.filter(item => !isMesh(item)); const meshImages: MeshImages = {}; diff --git a/cli/src/commands/sre-authority.ts b/cli/src/commands/sre-authority.ts new file mode 100644 index 000000000..5609fb5b1 --- /dev/null +++ b/cli/src/commands/sre-authority.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Command } from "commander"; +import { execa } from "execa"; +import { requireBundledAsset } from "../lib/repo-assets.js"; +import { enroll, preview, requireRegistrar, retire, waitForAuthority, type Execute } from "../lib/sre-authority.js"; +import { stageSource } from "../lib/sre-source.js"; +import { stageAuthority } from "../lib/sre-stage.js"; + +function executor(context?: string): Execute { + return (file, args, options) => execa(file, [ + ...(context ? [file === "helm" ? "--kube-context" : "--context", context] : []), ...args, + ], options); +} + +export function authorityCommand(): Command { + const command = new Command("authority").description("Stage, review, enroll and retire cluster-authorized SRE privacy"); + const common = (name: string) => command.command(name) + .option("--namespace ", "Controller/release namespace", "kars-system") + .option("--release ", "Owning Helm release", "kars") + .option("--context ", "Kubernetes context"); + common("stage") + .description("Explicit registrar staging: install authority APIs/controller while retaining legacy grants unchanged") + .requiredOption("--controller-image ", "Qualified prerequisite controller repository:tag") + .requiredOption("--router-image ", "Qualified prerequisite router repository:tag") + .option("--dry-run", "Server-side preview without deployment changes") + .action(async options => { + const execute = executor(options.context); + await stageAuthority(execute,requireBundledAsset("deploy/helm/kars"),options.namespace,options.release, + options.controllerImage,options.routerImage,!!options.dryRun); + console.log("Authority controller staged. Preview and explicitly enroll the exact SRE source/grants before normal upgrades."); + }); + common("preview").description("Read exact enrollment identities and legacy grants; no mutations") + .action(async options => { + const spec = await preview(executor(options.context),options.namespace,options.release); + console.log(JSON.stringify(spec,null,2)); + for (const binding of spec.legacyBindings) { + console.log(`--binding '${binding.kind}/${binding.namespace ?? ""}/${binding.name}=${binding.uid}@${binding.resourceVersion}'`); + } + if (spec.legacyConsumer) console.log(`--consumer '${spec.legacyConsumer.uid}@${spec.legacyConsumer.resourceVersion}'`); + }); + common("stage-source").description("Atomically create a genuinely new, unprivileged SRE source and wait for its exact claim") + .option("--model ", "SRE model deployment") + .action(async options => { + const execute=executor(options.context); + await requireRegistrar(execute); + const result=await execute("helm",["template",options.release,requireBundledAsset("deploy/helm/kars"), + "--namespace",options.namespace,"--show-only","templates/sre.yaml", + "--set","sre.enabled=true","--set","azure.workloadIdentity.clientId=dummy", + ...(options.model?["--set-string",`sre.model=${options.model}`]:[])],{stdio:"pipe"}); + const created=await stageSource(execute,result.stdout,options.namespace,options.release); + console.log(`Created and claimed source: --sandbox-uid ${created.uid} --namespace-uid ${created.namespaceUid}`); + }); + common("enroll").description("Enroll only reviewed source/namespace and grant UIDs under cluster registrar authority") + .requiredOption("--sandbox-uid ", "Reviewed canonical Sandbox UID") + .requiredOption("--namespace-uid ", "Reviewed claimed runtime namespace UID") + .option("--binding ", "Exact binding review from preview; repeat per binding", (value: string, all: string[]) => [...all,value], []) + .option("--consumer ", "Reviewed legacy SRE Deployment") + .option("--registration-uid ", "Required when updating an existing registration") + .option("--resource-version ", "Required when updating an existing registration") + .option("--dry-run", "Print the enrollment without writing it") + .action(async options => { + const execute = executor(options.context); + const spec = await preview(execute,options.namespace,options.release); + console.log(await enroll(execute,spec,options,!!options.dryRun)); + }); + common("migrate").description("Wait for the controller to complete the explicitly enrolled migration") + .action(async options => { + await requireRegistrar(executor(options.context)); + await waitForAuthority(executor(options.context),"Ready"); + console.log("SRE authority Ready: legacy credentials denied and private renewable identity configured."); + }); + common("retire").description("Disable and retire a reviewed registration before SRE uninstall") + .requiredOption("--registration-uid ", "Reviewed registration UID") + .requiredOption("--resource-version ", "Reviewed registration resourceVersion") + .action(async options => { + await retire(executor(options.context),options.registrationUid,options.resourceVersion); + console.log("SRE authority Retired; owned private grants and credentials are revoked."); + }); + return command; +} diff --git a/cli/src/commands/sre.test.ts b/cli/src/commands/sre.test.ts index 436c98fa5..ab72ec438 100644 --- a/cli/src/commands/sre.test.ts +++ b/cli/src/commands/sre.test.ts @@ -13,16 +13,76 @@ vi.mock("../lib/repo-assets.js", () => ({ requireBundledAsset: () => "/test/char const releases = JSON.stringify([{ name: "kars", namespace: "kars-system" }]); const controller = JSON.stringify({ apiVersion: "apps/v1", kind: "Deployment", - metadata: { name: "kars-controller", namespace: "kars-system", uid: "controller-uid" }, + metadata: { name: "kars-controller", namespace: "kars-system", uid: "controller-uid", resourceVersion: "1" }, }); +function authority(args: readonly string[]): { stdout: string } { + const metadata = (name: string, uid = name) => ({ name, uid, resourceVersion: "1", generation: 1 }); + const getIndex = args.indexOf("get"); + const resource = getIndex < 0 ? undefined : args[getIndex + 1]; + const name = getIndex < 0 ? undefined : args[getIndex + 2]; + let value: unknown; + if (args.includes("can-i")) return { stdout: "yes" }; + if (resource === "deployment" && name === "kars-controller") value = JSON.parse(controller); + if (resource === "crd") value = { metadata: metadata("karssreregistrations.kars.azure.com") }; + if (resource === "karssreregistrations.kars.azure.com") value = { + metadata: metadata("canonical"), + spec: { + enabled: true, + controller: { namespace: { name: "kars-system", uid: "kars-system" }, + deployment: { name: "kars-controller", uid: "controller-uid" }, release: currentRelease }, + sandbox: { namespace: "kars-system", name: "sre", uid: "source" }, + runtimeNamespace: { name: "kars-sre", uid: "kars-sre" }, + }, + status: { phase: "Ready", observedGeneration: 1, privacyRevision: "kars.azure.com/sre-privacy/v2" }, + }; + if (resource === "namespace") value = name === "kars-sre" + ? { metadata: { ...metadata("kars-sre"), annotations: { + "kars.azure.com/namespace-claim-version": "v1", + "kars.azure.com/sandbox-namespace": "kars-system", + "kars.azure.com/sandbox-name": "sre", + "kars.azure.com/sandbox-uid": "source", + } } } + : { metadata: metadata("kars-system") }; + if (resource === "karssandbox") value = { + metadata: { ...metadata("sre", "source"), namespace: "kars-system", + annotations: { "kars.azure.com/namespace-uid": "kars-sre" } }, + }; + if (resource === "clusterrolebindings" || resource === "rolebindings") value = { items: [] }; + return { stdout: value ? JSON.stringify(value) : "" }; +} + +let currentRelease = "kars"; + beforeEach(() => { + currentRelease = "kars"; execute.mockReset(); vi.spyOn(console, "log").mockImplementation(() => {}); }); afterEach(() => vi.restoreAllMocks()); describe("SRE controller upgrade namespace preflight", () => { + it("retains the existing release when Helm 4 removes the all-status flag", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + execute.mockImplementation(async (file, args) => { + if (file === "helm" && args[0] === "list") { + if (args.includes("--all")) { + throw Object.assign(new Error("Unsupported flag"), { + exitCode: 1, stderr: "Error: unknown flag: --all\n", + }); + } + expect(args).toContain("--kube-context"); + return { stdout: releases }; + } + if (file === "helm" && args[0] === "version") return { stdout: "v4.2.4" }; + if (file === "kubectl" && args.includes("karssandboxes")) return { stdout: '{"items":[]}' }; + return authority(args); + }); + await sreCommand().parseAsync(["node", "sre", "install", "--no-wait", "--context", "test-context"]); + expect(execute.mock.calls.some(([file, args]) => file === "helm" && args[0] === "upgrade")).toBe(true); + expect(execute.mock.calls.some(([file, args]) => file === "helm" && args[0] === "install")).toBe(false); + }); + it.each(["upgrade", "template"])("preflights %s mode in the selected context before mutations", async mode => { execute.mockImplementation(async (file, args) => { if (file === "helm" && args[0] === "list") return { stdout: mode === "upgrade" ? releases : "[]" }; @@ -34,7 +94,7 @@ describe("SRE controller upgrade namespace preflight", () => { expect(args.slice(0, 2)).toEqual(["--context", "test-context"]); return { stdout: '{"items":[]}' }; } - return { stdout: "" }; + return authority(args); }); await sreCommand().parseAsync(["node", "sre", "install", "--no-wait", "--context", "test-context"]); const calls = execute.mock.calls; @@ -89,14 +149,22 @@ describe("SRE controller upgrade namespace preflight", () => { }); it("permits fresh installation only after successful inventory and explicit controller absence", async () => { - execute.mockImplementation(async (file, args) => ({ - stdout: file === "helm" && args[0] === "list" ? "[]" : "", - })); + execute.mockImplementation(async (file, args) => { + if (file === "helm" && args[0] === "list") return { stdout: "[]" }; + if (args[0] === "-n") return { stdout: "" }; + if (args.includes("deployment") && args.includes("kars-controller")) return { stdout: controller }; + return authority(args); + }); await sreCommand().parseAsync(["node", "sre", "install", "--no-wait"]); - expect(execute.mock.calls.map(([file, args]) => [file, args[0]])).toEqual([ - ["helm", "list"], ["kubectl", "-n"], ["helm", "install"], + expect(execute.mock.calls.slice(0, 2).map(([file, args]) => [file, args[0]])).toEqual([ + ["helm", "list"], ["kubectl", "-n"], ]); expect(execute.mock.calls[1][1]).toContain("--ignore-not-found"); + const installs = execute.mock.calls.filter(([file, args]) => file === "helm" && args[0] === "install"); + expect(installs).toHaveLength(1); + expect(installs[0][1]).toContain("sre.enabled=false"); + expect(execute.mock.calls.some(([, args]) => args.includes("--take-ownership") || args.includes("--force-conflicts"))).toBe(false); + expect(execute.mock.calls.some(([file, args]) => file === "helm" && args[0] === "upgrade" && args.includes("sre.enabled=true"))).toBe(true); }); it.each([ @@ -105,13 +173,14 @@ describe("SRE controller upgrade namespace preflight", () => { { target: "kars", names: ["other.release"] }, { target: "a".repeat(53), names: ["a".repeat(53)] }, ])("accepts Helm-compatible release inventory for $target", async ({ target, names }) => { + currentRelease = target; execute.mockImplementation(async (file, args) => { if (file === "helm" && args[0] === "list") { return { stdout: JSON.stringify(names.map(name => ({ name, namespace: "kars-system" }))) }; } - if (file === "kubectl" && args.includes("deployment")) return { stdout: controller }; + if (file === "kubectl" && args.includes("deployment") && args.includes("kars-controller")) return { stdout: controller }; if (file === "kubectl" && args.includes("karssandboxes")) return { stdout: '{"items":[]}' }; - return { stdout: "" }; + return authority(args); }); await sreCommand().parseAsync(["node", "sre", "install", "--no-wait", "--release", target]); const mode = names.includes(target) ? "upgrade" : "template"; diff --git a/cli/src/commands/sre.ts b/cli/src/commands/sre.ts index 09e8e71e6..166d31321 100644 --- a/cli/src/commands/sre.ts +++ b/cli/src/commands/sre.ts @@ -6,6 +6,10 @@ import chalk from "chalk"; import { execa } from "execa"; import { requireBundledAsset } from "../lib/repo-assets.js"; import { inspectNamespaceOwnership } from "../lib/namespace-ownership.js"; +import { authorityCommand } from "./sre-authority.js"; +import { assertDestroySafe, assertSafeMutation, enroll, get, preview, registration, requireRegistrar, waitForAuthority } from "../lib/sre-authority.js"; +import { stageSource } from "../lib/sre-source.js"; +import { listSreHelmReleases } from "../lib/sre-helm.js"; const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; @@ -27,6 +31,7 @@ const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z export function sreCommand(): Command { const cmd = new Command("sre"); cmd.description("Manage the built-in kars-sre agent (Kubernetes SRE on the cluster)"); + cmd.addCommand(authorityCommand()); cmd .command("install") @@ -74,15 +79,13 @@ export function sreCommand(): Command { // B. operator deployed via `kars dev --target local-k8s` // (which renders `helm template | kubectl apply` and so // never creates a helm release record) → use `helm template - // | kubectl apply --server-side --force-conflicts` with - // `sre.enabled=true` baked in. The chart is already in - // the cluster; this just adds the SRE bits idempotently. - // C. no chart at all → `helm install` with --take-ownership + - // a fallback workload-identity client-id (local dev). + // | kubectl apply --server-side` without force adoption. + // C. no chart at all → install unprivileged core first, then + // atomically create and enroll the fresh SRE source. let mode: "upgrade" | "template" | "install" = "install"; - const listArgs = ["list", "-n", options.namespace, "--all", "-o", "json"]; - if (options.context) listArgs.push("--kube-context", options.context); - const { stdout: releasesOutput } = await execa("helm", listArgs, { stdio: "pipe", timeout: 30_000 }); + const releasesOutput = await listSreHelmReleases((file, args, commandOptions) => + execa(file, [...args, ...(options.context ? ["--kube-context", options.context] : [])], + { ...commandOptions, timeout: 30_000 }), options.namespace); const releases: unknown = JSON.parse(releasesOutput); if (!Array.isArray(releases) || releases.some(release => !release || typeof release !== "object" || typeof release.name !== "string" @@ -124,6 +127,43 @@ export function sreCommand(): Command { execa(file, [...(options.context ? ["--context", options.context] : []), ...args], commandOptions)); } + const execute = (file: string, args: readonly string[], commandOptions: { stdio: "pipe"; input?: string }) => + execa(file, [...(options.context ? [file === "helm" ? "--kube-context" : "--context", options.context] : []), ...args], commandOptions); + await requireRegistrar(execute); + if (mode === "install") { + await execute("helm", ["install", options.release, chartPath, "--namespace", options.namespace, + "--create-namespace", "--set", "sre.enabled=false", + "--set", "azure.workloadIdentity.clientId=dummy", "--wait", "--timeout", "8m"], { stdio: "pipe" }); + mode = "upgrade"; + } + if (!await get(execute, "crd", "karssreregistrations.kars.azure.com")) { + throw new Error("Stage the SRE authority prerequisite controller/APIs before installing SRE"); + } + const enrolled = await registration(execute); + if (enrolled) { + if (enrolled.spec?.controller?.namespace?.name !== options.namespace + || enrolled.spec?.controller?.release !== options.release) { + throw new Error("Canonical SRE is registered to another source/release; no privilege was granted"); + } + if (enrolled.spec?.enabled === false) { + throw new Error("Retired SRE authority must be explicitly re-enrolled with reviewed current UIDs before install"); + } + await assertSafeMutation(execute); + } else { + const rendered = await execute("helm", ["template", options.release, chartPath, + "--namespace", options.namespace, "--show-only", "templates/sre.yaml", + "--set", "sre.enabled=true", "--set", "azure.workloadIdentity.clientId=dummy", + ...(options.model ? ["--set-string", `sre.model=${options.model}`] : [])], { stdio: "pipe" }); + const created = await stageSource(execute, rendered.stdout, options.namespace, options.release); + const spec = await preview(execute, options.namespace, options.release); + if (spec.legacyBindings.length) throw new Error("A legacy grant appeared during staging; explicit review is required"); + if (spec.legacyConsumer) throw new Error("An SRE consumer appeared during staging; explicitly review its UID/resourceVersion before enrollment"); + await enroll(execute, spec, { + sandboxUid: created.uid, namespaceUid: created.namespaceUid, binding: [], + }, false); + await waitForAuthority(execute, "Ready"); + } + const helmArgs = mode === "upgrade" ? [ @@ -137,14 +177,8 @@ export function sreCommand(): Command { // release values predate fields like runtimes.hermes — a plain // --reuse-values would carry the gap forward and fail templating. "--reset-then-reuse-values", - // --force-conflicts: helm 4 uses server-side apply by default, - // which conflicts with field managers from prior `kubectl set - // image` / `kars push --apply` runs that touched the same - // fields. This flag tells SSA to take ownership on conflict, - // matching the operator's intent (helm-managed chart is the - // source of truth). - "--force-conflicts", "--set", "sre.enabled=true", + "--set", "sre.authorityStage=false", ] : mode === "template" ? [ @@ -165,15 +199,6 @@ export function sreCommand(): Command { chartPath, "--namespace", options.namespace, "--create-namespace", - "--force-conflicts", - // --take-ownership: adopt resources that already exist in the - // cluster but don't carry helm metadata (the kars-system - // namespace, default-deny NetworkPolicy, etc. created - // out-of-band by a prior `kars dev` or partial helm - // install). Without this, install dies on the first such - // resource with a "cannot be imported" error. Requires - // helm >= 3.17 (`kars dev` pins helm 4 — safe). - "--take-ownership", "--set", "sre.enabled=true", // Brand-new chart install on a fresh cluster has no prior // azure.workloadIdentity.clientId — use a dummy fallback for @@ -205,7 +230,6 @@ export function sreCommand(): Command { "apply", "-f", "-", "--server-side", - "--force-conflicts", ], { input: stdout, @@ -280,9 +304,11 @@ export function sreCommand(): Command { chartPath, "--namespace", options.namespace, "--reset-then-reuse-values", - "--force-conflicts", "--set", "sre.enabled=false", ]; + await assertDestroySafe((file,args,commandOptions) => execa(file,[ + ...(options.context ? [file === "helm" ? "--kube-context" : "--context",options.context] : []),...args, + ],commandOptions)); if (options.context) helmArgs.push("--kube-context", options.context); console.log(chalk.cyan("▸ disabling kars-sre via helm upgrade --reuse-values…")); diff --git a/cli/src/commands/up.ts b/cli/src/commands/up.ts index c958dd2a3..708cf91a9 100644 --- a/cli/src/commands/up.ts +++ b/cli/src/commands/up.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { Command } from "commander"; +import { assertSafeMutation } from "../lib/sre-authority.js"; import chalk from "chalk"; import { existsSync } from "fs"; import * as path from "path"; @@ -605,6 +606,7 @@ Auto-resume: ], { stdio: "pipe" }); stepper.done("kubectl configured"); markPhaseDone("kubectl", {}, resumeTopology); + await assertSafeMutation(execa); // ── Step 6: Get images into ACR ────────────────────────────── const acr = acrLoginServer.replace(".azurecr.io", ""); @@ -782,11 +784,8 @@ Auto-resume: // small system nodes. 10m avoids a spurious "context deadline // exceeded" while k8s is still legitimately rolling out. "--timeout", "10m", - // Take ownership of fields previously written by `kubectl apply` - // or `kubectl patch` (e.g. CRDs / ClusterRoles touched out-of-band - // during prior debugging). Without this, Helm's server-side apply - // refuses with "conflict with kubectl-client-side-apply" and the - // whole `kars up` flow fails after the 18-min image build. + // Preserve the existing core field-manager behavior; the SRE + // preflight above rejects unreviewed grant migration before this. "--force-conflicts", ]; if (foundryEndpoint) { diff --git a/cli/src/commands/up/fast_upgrade.ts b/cli/src/commands/up/fast_upgrade.ts index a65e67440..daa621f3a 100644 --- a/cli/src/commands/up/fast_upgrade.ts +++ b/cli/src/commands/up/fast_upgrade.ts @@ -14,6 +14,7 @@ import { requireBundledAsset } from "../../lib/repo-assets.js"; import { cliReleaseTag } from "../../lib/version.js"; import { rolloutRestartAll } from "../upgrade.js"; import { inspectNamespaceOwnership } from "../../lib/namespace-ownership.js"; +import { assertSafeMutation } from "../../lib/sre-authority.js"; export interface UpOptionsForUpgrade { upgrade?: boolean; @@ -38,6 +39,7 @@ export async function runFastUpgrade(options: UpOptionsForUpgrade): Promise ({ name, uid: name, resourceVersion: "1" }); + +function cluster(name: string) { + return { id: `${groupId}/providers/Microsoft.ContainerService/managedClusters/${name}`, + name, resourceGroup: group, resourceUid: `${name}-immutable-resource-uid` }; +} +function credentials(name: string) { + return JSON.stringify({ apiVersion: "v1", kind: "Config", "current-context": name, + contexts: [{ name, context: { cluster: name, user: name } }], + clusters: [{ name, cluster: endpoint }], users: [{ name, user: { token: "synthetic-test-token" } }] }); +} +function fixture(names = ["cluster-b"]) { + const state = { + clusters: names.map(cluster), inventoryCalls: 0, active: new Set(), unavailable: "", + selectedUid: "cluster-b-kube-system", selectedEndpoint: { ...endpoint }, + changedInventory: undefined as undefined | ReturnType[], + }; + const files = new Set(); + const execute = vi.fn(async (file, args) => { + if (file === "az") { + if (args[0] === "account") return { stdout: subscription }; + expect(args.slice(-2)).toEqual(["--subscription", subscription]); + if (args[0] === "group" && args[1] === "show") return { stdout: groupId }; + if (args[0] === "aks" && args[1] === "list") { + state.inventoryCalls++; + return { stdout: JSON.stringify(state.inventoryCalls > 1 ? state.changedInventory ?? state.clusters : state.clusters) }; + } + if (args[0] === "rest") { + expect(args.slice(0, 3)).toEqual(["rest", "--method", "post"]); + const url = args[args.indexOf("--url") + 1]; + const target = state.clusters.find(cluster => + url === `${cluster.id.toLowerCase()}/listClusterUserCredential?api-version=2024-10-01`); + expect(target).toBeDefined(); + const name = target!.name; + if (state.unavailable === name) throw new Error("AKS credentials Forbidden"); + return { stdout: JSON.stringify({ kubeconfigs: [{ name: "clusterUser", value: Buffer.from(credentials(name)).toString("base64") }] }) }; + } + if (args[0] === "group" && args[1] === "delete") return { stdout: "" }; + } + if (file === "kubectl") { + const context = args[args.indexOf("--context") + 1]; + if (args.includes("config")) return { stdout: JSON.stringify(state.selectedEndpoint) }; + if (args.includes("--kubeconfig")) { + const path = args[args.indexOf("--kubeconfig") + 1]; + files.add(path); + expect(statSync(dirname(path)).mode & 0o777).toBe(0o700); + const descriptor = openSync(path, "r"); + try { + expect(fstatSync(descriptor).mode & 0o777).toBe(0o600); + expect(JSON.parse(readFileSync(descriptor, "utf8"))["current-context"]).toBe(context); + } finally { + closeSync(descriptor); + } + } + const start = args.indexOf("get"); + const [kind, name] = args.slice(start + 1, start + 3); + if (kind === "namespace") return { stdout: name === "kube-system" ? JSON.stringify({ + metadata: { ...metadata(name), uid: context === "selected" ? state.selectedUid : `${context}-kube-system` }, + }) : "" }; + if (kind === "crd") return { stdout: state.active.has(context) ? JSON.stringify({ metadata: metadata(name) }) : "" }; + if (kind === "karssreregistrations.kars.azure.com") return { stdout: JSON.stringify({ + metadata: { ...metadata("canonical"), generation: 1 }, spec: { enabled: true }, + status: { phase: "Ready", observedGeneration: 1 }, + }) }; + } + throw new Error(`Unexpected fixture command: ${file} ${args.join(" ")}`); + }); + const remove = vi.fn(async (azure: Execute) => { + await azure("az", ["group", "delete", "--name", group, "--yes", "--no-wait", "--output", "none"], { stdio: "pipe" }); + }); + const run = (context?: string, exec: Execute = execute, sub?: string) => withAzureDestroyTarget(exec, group, sub, context, remove); + const cleaned = () => { + for (const path of files) { + expect(existsSync(path)).toBe(false); + expect(existsSync(dirname(path))).toBe(false); + } + }; + return { state, execute, files, remove, run, cleaned }; +} + +describe("Azure teardown retirement target binding", () => { + it("checks ALL resource-group AKS APIs, ignores global current-context and pins subscription through deletion", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + await f.run(); + expect(f.remove).toHaveBeenCalledOnce(); + expect(f.state.inventoryCalls).toBe(2); + expect(f.files.size).toBe(2); + expect(f.execute.mock.calls.some(([file, args]) => file === "kubectl" && !args.includes("--kubeconfig"))).toBe(false); + expect(f.execute.mock.calls.some(([, args]) => args.includes("current-context") || args.includes("--overwrite-existing"))).toBe(false); + expect(f.execute.mock.calls.some(([, args]) => args.includes("get-credentials") || args.includes("convert-kubeconfig"))).toBe(false); + const deletion = f.execute.mock.calls.find(([, args]) => args[0] === "group" && args[1] === "delete")!; + expect(deletion[1].slice(-2)).toEqual(["--subscription", subscription]); + f.cleaned(); + }); + + it("resolves an explicit subscription once and never follows a later selected-account switch", async () => { + const f = fixture(); + await f.run(undefined, f.execute, subscription); + expect(f.execute.mock.calls[0][1]).toEqual(["account", "show", "--subscription", subscription, "--query", "id", "--output", "tsv"]); + expect(f.execute.mock.calls.filter(([, args]) => args[0] === "account")).toHaveLength(1); + f.cleaned(); + }); + + it("accepts an explicit context only with the ARM credential TLS endpoint AND real cluster UID", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + await f.run("selected"); + expect(f.remove).toHaveBeenCalledOnce(); + expect(f.files.size).toBe(2); + f.cleaned(); + }); + + it.each(["uid", "ca", "server"])("rejects context A versus deletion target B (%s mismatch)", async field => { + const f = fixture(); + if (field === "uid") f.state.selectedUid = "cluster-a-kube-system"; + if (field === "ca") f.state.selectedEndpoint["certificate-authority-data"] = Buffer.concat([ca, Buffer.from("\n")]).toString("base64"); + if (field === "server") f.state.selectedEndpoint.server = "https://different.example.test/"; + await expect(f.run("selected")).rejects.toThrow("--context does not match"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it("blocks an active registration in a second cluster even when the selected context is retired/empty", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + f.state.active.add("cluster-c"); + await expect(f.run("selected")).rejects.toThrow("Retire"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it("fails closed when any cluster's credentials/API are inaccessible", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + f.state.unavailable = "cluster-c"; + await expect(f.run()).rejects.toThrow("Cannot obtain AKS user credentials"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it.each(["new-cluster", "removed-cluster", "replaced-cluster"])("blocks a %s inventory race before deletion", async change => { + const f = fixture(); + f.state.changedInventory = change === "new-cluster" ? [...f.state.clusters, cluster("new")] + : change === "removed-cluster" ? [] : [{ ...f.state.clusters[0], resourceUid: "replacement-uid" }]; + await expect(f.run()).rejects.toThrow("inventory changed"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it.each(["id", "name", "resourceUid", "resourceGroup"])("rejects ambiguous or foreign ARM inventory %s", async key => { + const f = fixture(); + (f.state.clusters[0] as any)[key] = key === "id" ? f.state.clusters[0].id.replace(subscription, otherSubscription) : ""; + await expect(f.run()).rejects.toThrow("exact ARM/resourceUid identity"); + expect(f.remove).not.toHaveBeenCalled(); + }); + + it("rejects duplicate clusters rather than treating repeated names as distinct proof", async () => { + const f = fixture(); + f.state.clusters.push({ ...f.state.clusters[0] }); + await expect(f.run()).rejects.toThrow("exact ARM/resourceUid identity"); + expect(f.remove).not.toHaveBeenCalled(); + }); + + it.each(["empty", "duplicate", "bad-base64", "bad-yaml", "insecure-tls"])( + "rejects %s ARM credentials without leaking their response", async kind => { + const f = fixture(); + const marker = "credential-response-must-not-be-printed"; + let value: any = { kubeconfigs: [] }; + if (kind === "duplicate") value.kubeconfigs = [{ value: marker }, { value: marker }]; + if (kind === "bad-base64") value.kubeconfigs = [{ value: marker }]; + if (kind === "bad-yaml") value.kubeconfigs = [{ value: Buffer.from(`secret: [${marker}`).toString("base64") }]; + if (kind === "insecure-tls") { + const config = JSON.parse(credentials("cluster-b")); + config.clusters[0].cluster["insecure-skip-tls-verify"] = true; + value.kubeconfigs = [{ value: Buffer.from(JSON.stringify(config)).toString("base64") }]; + } + const execute: Execute = (file, args, options) => args[0] === "rest" + ? Promise.resolve({ stdout: JSON.stringify(value) }) : f.execute(file, args, options); + let error: unknown; + try { await f.run(undefined, execute); } catch (caught) { error = caught; } + expect(error).toBeInstanceOf(Error); + expect(String(error)).not.toContain(marker); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }, + ); + + it.each(["account", "group", "list", "api"])("does not interpret a %s failure as cluster absence", async point => { + const f = fixture(); + const execute: Execute = (file, args, options) => (point === "account" && args[0] === "account") + || (point === "group" && args[0] === "group" && args[1] === "show") + || (point === "list" && args[1] === "list") || (point === "api" && file === "kubectl") + ? Promise.reject(new Error("Forbidden target proof")) : f.execute(file, args, options); + await expect(f.run(undefined, execute)).rejects.toThrow("Forbidden target proof"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it("permits an authoritatively empty AKS inventory, but cannot claim an explicit unrelated context", async () => { + const f = fixture([]); + await f.run(); + expect(f.remove).toHaveBeenCalledOnce(); + expect(f.state.inventoryCalls).toBe(2); + f.remove.mockClear(); + await expect(f.run("selected")).rejects.toThrow("--context does not match"); + expect(f.remove).not.toHaveBeenCalled(); + }); + + it("rejects subscription retargeting even inside the destructive callback", async () => { + const f = fixture(); + await expect(withAzureDestroyTarget(f.execute, group, undefined, undefined, async azure => { + await azure("az", ["group", "delete", "--name", group, "--subscription", otherSubscription], { stdio: "pipe" }); + })).rejects.toThrow("not the deployment subscription"); + expect(f.execute.mock.calls.some(([, args]) => args[1] === "delete")).toBe(false); + f.cleaned(); + }); +}); diff --git a/cli/src/lib/azure-destroy-target.ts b/cli/src/lib/azure-destroy-target.ts new file mode 100644 index 000000000..0452a2c63 --- /dev/null +++ b/cli/src/lib/azure-destroy-target.ts @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID, X509Certificate } from "node:crypto"; +import { mkdir, open, unlink, rmdir } from "node:fs/promises"; +import { resolve } from "node:path"; +import { parse } from "yaml"; +import { pinAzureSubscription } from "./azure-subscription.js"; +import { assertDestroySafe, get, type Execute } from "./sre-authority.js"; + +interface Cluster { id: string; name: string; resourceGroup: string; resourceUid: string } + +function trustedEndpoint(cluster: any): string { + if (!cluster || typeof cluster.server !== "string" + || typeof cluster["certificate-authority-data"] !== "string" + || cluster["insecure-skip-tls-verify"] || cluster["proxy-url"] || cluster["tls-server-name"] + || cluster["certificate-authority"]) throw new Error("AKS target lacks an unambiguous TLS-authenticated API endpoint"); + const url = new URL(cluster.server); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash || url.pathname !== "/") { + throw new Error("Invalid AKS API endpoint"); + } + const ca = Buffer.from(cluster["certificate-authority-data"], "base64"); + new X509Certificate(ca); + return `${url.href}\n${createHash("sha256").update(ca).digest("hex")}`; +} + +function credentialIdentity(stdout: string): { context: string; endpoint: string } { + let config; + try { config = parse(stdout); } catch { + throw new Error("Azure returned malformed AKS credentials; deletion is blocked"); + } + const context = config?.["current-context"]; + if (typeof context !== "string" || !context || !Array.isArray(config.contexts) || !Array.isArray(config.clusters)) { + throw new Error("Azure returned an invalid AKS kubeconfig"); + } + const contexts = config.contexts.filter((item: any) => item.name === context); + if (contexts.length !== 1) throw new Error("Azure returned an ambiguous AKS context"); + const clusters = config.clusters.filter((item: any) => item.name === contexts[0].context?.cluster); + if (clusters.length !== 1) throw new Error("Azure returned an ambiguous AKS cluster"); + return { context, endpoint: trustedEndpoint(clusters[0].cluster) }; +} + +async function userCredentials(azure: Execute, id: string): Promise { + // Some az aks get-credentials versions run kubelogin conversion even with + // --file -, potentially touching the global kubeconfig. Read ARM directly. + let result; + try { + const response = await azure("az", ["rest", "--method", "post", "--url", + `${id}/listClusterUserCredential?api-version=2024-10-01`, "--output", "json"], { stdio: "pipe" }); + result = JSON.parse(response.stdout); + } catch { + // Command/YAML errors must never print credential-bearing response bodies. + throw new Error("Cannot obtain AKS user credentials from the selected ARM resource; deletion is blocked"); + } + const configs = result?.kubeconfigs; + if (!Array.isArray(configs) || configs.length !== 1 || typeof configs[0]?.value !== "string" + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(configs[0].value) + || !configs[0].value) throw new Error("Azure returned ambiguous or invalid AKS credentials; deletion is blocked"); + return Buffer.from(configs[0].value, "base64").toString("utf8"); +} + +/** Bind every retirement check to ARM-issued credentials for every AKS resource + * in the deletion target. Neither current-context nor an AKS name is proof. + * The callback receives the same subscription pin used by the entire preflight. */ +export async function withAzureDestroyTarget( + execute: Execute, resourceGroup: string, requestedSubscription: string | undefined, + requestedContext: string | undefined, destroy: (azure: Execute) => Promise, +): Promise { + const subscription = (await execute("az", ["account", "show", + ...(requestedSubscription ? ["--subscription", requestedSubscription] : []), + "--query", "id", "--output", "tsv"], { stdio: "pipe" })).stdout.trim(); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(subscription)) { + throw new Error("Azure did not resolve a subscription ID; resource-group deletion is blocked"); + } + const azure: Execute = (file, args, options) => { + if (file !== "az") throw new Error("Azure teardown executor cannot use an unbound Kubernetes context"); + return execute(file, pinAzureSubscription(args, subscription), options); + }; + const groupId = `/subscriptions/${subscription}/resourceGroups/${resourceGroup}`; + async function inventory(): Promise { + const group = (await azure("az", ["group", "show", "--name", resourceGroup, + "--query", "id", "--output", "tsv"], { stdio: "pipe" })).stdout.trim(); + if (group.toLowerCase() !== groupId.toLowerCase()) throw new Error("Azure resource-group target identity mismatch"); + const result: unknown = JSON.parse((await azure("az", ["aks", "list", "--resource-group", resourceGroup, + "--output", "json"], { stdio: "pipe" })).stdout); + if (!Array.isArray(result)) throw new Error("Azure AKS inventory is malformed; deletion is blocked"); + const seen = new Set(); + const clusters: Cluster[] = []; + for (const cluster of result) { + if (!cluster || typeof cluster.name !== "string" || !cluster.name + || typeof cluster.id !== "string" || typeof cluster.resourceGroup !== "string" + || typeof cluster.resourceUid !== "string" || !cluster.resourceUid.trim() + || cluster.resourceGroup.toLowerCase() !== resourceGroup.toLowerCase() + || cluster.id.toLowerCase() !== `${groupId}/providers/Microsoft.ContainerService/managedClusters/${cluster.name}`.toLowerCase() + || seen.has(cluster.id.toLowerCase())) { + throw new Error("AKS inventory lacks exact ARM/resourceUid identity; update Azure CLI or review the target before deletion"); + } + seen.add(cluster.id.toLowerCase()); + clusters.push({ id: cluster.id.toLowerCase(), name: cluster.name, resourceGroup: cluster.resourceGroup, + resourceUid: cluster.resourceUid }); + } + return clusters.sort((a, b) => a.id.localeCompare(b.id)); + } + const clusters = await inventory(); + let selected: { endpoint: string; uid: string } | undefined; + if (requestedContext) { + const endpoint = trustedEndpoint(JSON.parse((await execute("kubectl", ["--context", requestedContext, + "config", "view", "--minify", "--raw", "-o", "jsonpath={.clusters[0].cluster}"], { stdio: "pipe" })).stdout)); + const ns = await get((file, args, options) => + execute(file, ["--context", requestedContext, ...args], options), "namespace", "kube-system"); + if (!ns) throw new Error("Selected context lacks a live kube-system identity"); + selected = { endpoint, uid: ns.metadata.uid! }; + } + let contextMatched = !selected; + const directory = resolve(`.kars-destroy-${randomUUID()}`); + await mkdir(directory, { mode: 0o700 }); + const owned: string[] = []; + try { + for (const [index, cluster] of clusters.entries()) { + const credentials = await userCredentials(azure, cluster.id); + const identity = credentialIdentity(credentials); + const path = resolve(directory, `cluster-${index}.yaml`); + const file = await open(path, "wx", 0o600); + owned.push(path); + try { await file.writeFile(credentials); } finally { await file.close(); } + const bound: Execute = (file, args, options) => { + if (file !== "kubectl") throw new Error("Retirement preflight requires the bound AKS API"); + return execute(file, ["--kubeconfig", path, "--context", identity.context, ...args], options); + }; + const before = await get(bound, "namespace", "kube-system"); + if (!before) throw new Error("AKS API lacks its live kube-system identity"); + if (selected?.endpoint === identity.endpoint && selected.uid === before.metadata.uid) contextMatched = true; + await assertDestroySafe(bound); + const after = await get(bound, "namespace", "kube-system"); + if (after?.metadata.uid !== before.metadata.uid) throw new Error("AKS API identity changed during retirement preflight"); + } + if (!contextMatched) throw new Error("--context does not match any TLS/UID-verified AKS cluster in the selected resource group"); + if (JSON.stringify(await inventory()) !== JSON.stringify(clusters)) { + throw new Error("Azure AKS inventory changed during retirement preflight; deletion is blocked"); + } + await destroy(azure); + } finally { + for (const path of owned) await unlink(path); + // No recursive deletion: foreign files are never removed. + await rmdir(directory); + } +} diff --git a/cli/src/lib/sre-action-crd.ts b/cli/src/lib/sre-action-crd.ts new file mode 100644 index 000000000..73fd402bd --- /dev/null +++ b/cli/src/lib/sre-action-crd.ts @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { get, type ApiObject, type Execute } from "./sre-authority.js"; + +export const ACTION_CRD = "karssreactions.kars.azure.com"; +const PARAMS = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/action/properties/params"; +// Complete immutable baseline spec, not a name/label-based adoption rule. +// Azure/kars 8b206065608593667a40665b3f48225ef9ce278d (BASE365). +const BASELINE = "2e119f9dee47426bdf1332e346a5f75f8b6a2d0d6c2a2781903406e7e25b577e"; + +function canonical(value: any): string { + const sort = (v: any): any => Array.isArray(v) ? v.map(sort) + : v && typeof v === "object" ? Object.fromEntries(Object.keys(v).sort().map(k => [k, sort(v[k])])) : v; + return JSON.stringify(sort(value)); +} + +function normalizedSpec(object: ApiObject): any { + const spec = structuredClone(object.spec); + if (!spec?.names || !Array.isArray(spec.versions)) throw new Error("Action CRD schema is missing"); + // Only API defaulting is ignored; custom fields, versions and validations + // are never overwritten or treated as the recognized historical schema. + spec.names.categories ??= []; + if (spec.names.listKind === "KarsSREActionList") delete spec.names.listKind; + if (canonical(spec.conversion) === '{"strategy":"None"}') delete spec.conversion; + if (spec.preserveUnknownFields === false) delete spec.preserveUnknownFields; + for (const version of spec.versions) { + if (version.deprecated === false) delete version.deprecated; + for (const column of version.additionalPrinterColumns ?? []) { + if (column.priority === 0) delete column.priority; + } + } + return spec; +} + +export async function planActionCrd( + execute: Execute, desired: ApiObject, namespace: string, release: string, helm: boolean, +): Promise<() => Promise> { + const spec = normalizedSpec(desired); + const legacy = structuredClone(spec); + const params = legacy.versions[0]?.schema?.openAPIV3Schema?.properties?.spec?.properties?.action?.properties?.params; + if (desired.kind !== "CustomResourceDefinition" || desired.metadata.name !== ACTION_CRD + || params?.["x-kubernetes-preserve-unknown-fields"] !== true || "additionalProperties" in params) { + throw new Error("Staged chart lacks the compatible action CRD params repair"); + } + delete params["x-kubernetes-preserve-unknown-fields"]; + params.additionalProperties = true; + if (createHash("sha256").update(canonical(legacy)).digest("hex") !== BASELINE) { + throw new Error("Unrecognized action CRD chart schema; explicitly review compatibility before staging policies"); + } + const existing = await get(execute, "customresourcedefinition", ACTION_CRD); + if (!existing) { + return async () => { + await execute("kubectl", ["create", "-f", "-"], { stdio: "pipe", input: JSON.stringify({ + ...desired, metadata: { ...desired.metadata, ...(helm ? { + labels: { ...desired.metadata.labels, "app.kubernetes.io/managed-by": "Helm" }, + annotations: { "meta.helm.sh/release-name": release, "meta.helm.sh/release-namespace": namespace }, + } : {}) }, + }) }); + await established(execute); + }; + } + const annotations = existing.metadata.annotations ?? {}; + const manager = existing.metadata.labels?.["app.kubernetes.io/managed-by"]; + if (existing.metadata.deletionTimestamp || existing.metadata.ownerReferences?.length + || (manager && manager !== "Helm") + || (annotations["meta.helm.sh/release-name"] && (!helm || annotations["meta.helm.sh/release-name"] !== release)) + || (annotations["meta.helm.sh/release-namespace"] && (!helm || annotations["meta.helm.sh/release-namespace"] !== namespace)) + || (annotations["kars.azure.com/sre-authority-staged"] && annotations["kars.azure.com/sre-authority-staged"] !== namespace) + || (annotations["kars.azure.com/sre-authority-release"] && annotations["kars.azure.com/sre-authority-release"] !== release) + || (manager === "Helm" && (!helm || annotations["meta.helm.sh/release-name"] !== release + || annotations["meta.helm.sh/release-namespace"] !== namespace))) { + throw new Error("Foreign or terminating action CRD; review its installation ownership before staging"); + } + const current = normalizedSpec(existing); + const repair = canonical(current) === canonical(legacy); + if (!repair && canonical(current) !== canonical(spec)) { + throw new Error("Customized action CRD schema; explicitly review the params compatibility repair before staging policies"); + } + return async () => { + // Even the already-repaired case tests the complete reviewed snapshot before + // installing dependent policies. No legacy ownership annotation is adopted. + await execute("kubectl", ["patch", "customresourcedefinition", ACTION_CRD, "--type=json", "-p", JSON.stringify([ + { op: "test", path: "/metadata/uid", value: existing.metadata.uid }, + { op: "test", path: "/metadata/resourceVersion", value: existing.metadata.resourceVersion }, + ...(repair ? [ + { op: "test", path: `${PARAMS}/additionalProperties`, value: true }, + { op: "remove", path: `${PARAMS}/additionalProperties` }, + { op: "add", path: `${PARAMS}/x-kubernetes-preserve-unknown-fields`, value: true }, + ] : []), + ])], { stdio: "pipe" }); + await established(execute); + }; +} + +async function established(execute: Execute): Promise { + await execute("kubectl", ["wait", "--for=condition=Established", `crd/${ACTION_CRD}`, "--timeout=60s"], { stdio: "pipe" }); +} diff --git a/cli/src/lib/sre-authority.test.ts b/cli/src/lib/sre-authority.test.ts new file mode 100644 index 000000000..90db36a16 --- /dev/null +++ b/cli/src/lib/sre-authority.test.ts @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe,expect,it,vi } from "vitest"; +import { assertDestroySafe,assertRollbackSafe,assertSafeMutation,enroll,preview,waitForAuthority,type Execute } from "./sre-authority.js"; +import { stageSource } from "./sre-source.js"; +import { stageAuthority } from "./sre-stage.js"; +import { readFileSync } from "node:fs"; + +const actionCrd=readFileSync(new URL("../../../deploy/helm/kars/templates/crd-karssreaction.yaml",import.meta.url),"utf8"); + +function fixture() { + const objects:Record={ + "namespace//kars-system":{kind:"Namespace",metadata:{name:"kars-system",uid:"control-ns",resourceVersion:"1"}}, + "deployment/kars-system/kars-controller":{kind:"Deployment",metadata:{name:"kars-controller",namespace:"kars-system",uid:"controller",resourceVersion:"1"}}, + "karssandbox/kars-system/sre":{kind:"KarsSandbox",metadata:{name:"sre",namespace:"kars-system",uid:"source",resourceVersion:"1", + annotations:{"kars.azure.com/namespace-uid":"runtime-ns"}}}, + "namespace//kars-sre":{kind:"Namespace",metadata:{name:"kars-sre",uid:"runtime-ns",resourceVersion:"1", + annotations:{"kars.azure.com/namespace-claim-version":"v1","kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre","kars.azure.com/sandbox-uid":"source"}}}, + "crd//karssreregistrations.kars.azure.com":{kind:"CustomResourceDefinition",metadata:{name:"karssreregistrations.kars.azure.com",uid:"crd",resourceVersion:"1"}}, + }; + const bindings:any[]=[]; + let use=true; + const execute=vi.fn(async (_file,args,options)=>{ + if(args[0]==="auth")return {stdout:use?"yes":"no"}; + if(args[0]==="get"&&(args[1]==="clusterrolebindings"||args[1]==="rolebindings")) + return {stdout:JSON.stringify({items:args[1]==="clusterrolebindings"?bindings:[]})}; + if(args[0]==="get"){ + const namespace=args.includes("-n")?args[args.indexOf("-n")+1]:""; + const object=objects[`${args[1]}/${namespace}/${args[2]}`]; + return {stdout:object?JSON.stringify(object):""}; + } + if(args[0]==="create"){ + const body=JSON.parse(options.input!); + objects["karssreregistrations.kars.azure.com//canonical"]={...body,metadata:{...body.metadata,uid:"registration",resourceVersion:"1",generation:1}}; + return {stdout:JSON.stringify(objects["karssreregistrations.kars.azure.com//canonical"])}; + } + return {stdout:""}; + }); + return {objects,bindings,execute,denyUse:()=>{use=false}}; +} + +describe("SRE cluster registrar boundary",()=>{ + it("previews only metadata and never enrolls based on labels or a foreign namespace",async()=>{ + const f=fixture(); + const result=await preview(f.execute,"kars-system","kars"); + expect(result.sandbox.uid).toBe("source"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + f.objects["namespace//kars-sre"].metadata.annotations["kars.azure.com/sandbox-namespace"]="foreign"; + await expect(preview(f.execute,"kars-system","kars")).rejects.toThrow(/different|foreign|claim/i); + }); + + it("requires explicit registrar permission and reviewed UIDs",async()=>{ + const f=fixture(); + const spec=await preview(f.execute,"kars-system","kars"); + f.denyUse(); + await expect(enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:[]},false)).rejects.toThrow("registrar"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="create"||args[0]==="patch")).toBe(false); + }); + + it("requires every legacy binding and exact resource version before enrollment",async()=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"legacy",uid:"binding",resourceVersion:"4"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"kars-sre-reader"}, + subjects:[{kind:"ServiceAccount",name:"sandbox",namespace:"kars-sre"}]}); + const spec=await preview(f.execute,"kars-system","kars"); + await expect(enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:[]},false)).rejects.toThrow("Every legacy binding"); + await enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:["ClusterRoleBinding//legacy=binding@4"]},false); + const body=JSON.parse(f.execute.mock.calls.find(([,args])=>args[0]==="create")![2].input!); + expect(body.kind).toBe("KarsSRERegistration"); + expect(body.metadata.namespace).toBeUndefined(); + }); + + it("blocks ordinary mutation and rollback while old grants or authority remain",async()=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"legacy",uid:"binding",resourceVersion:"1"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"kars-sre-reader"}, + subjects:[{kind:"ServiceAccount",name:"sandbox",namespace:"kars-sre"}]}); + await expect(assertSafeMutation(f.execute)).rejects.toThrow("Legacy SRE grants"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + f.bindings.length=0; + f.objects["karssreregistrations.kars.azure.com//canonical"]={metadata:{name:"canonical",uid:"registration",resourceVersion:"1",generation:1}, + spec:{enabled:true},status:{phase:"Ready",observedGeneration:1}}; + await expect(assertRollbackSafe(f.execute)).rejects.toThrow("Rollback"); + await expect(assertDestroySafe(f.execute)).rejects.toThrow("Retire"); + }); + + it("dry-run enrollment performs no mutation",async()=>{ + const f=fixture(); + const spec=await preview(f.execute,"kars-system","kars"); + const result=await enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:[]},true); + expect(JSON.parse(result).spec.sandbox.uid).toBe("source"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="create"||args[0]==="patch")).toBe(false); + }); + + it.each([false,true])("replaces the full reviewed re-enrollment spec (consumer present: %s)",async hasConsumer=>{ + const f=fixture(); + if(hasConsumer) f.objects["deployment/kars-sre/sre"]={kind:"Deployment", + metadata:{name:"sre",namespace:"kars-sre",uid:"new-consumer",resourceVersion:"9"}}; + const spec=await preview(f.execute,"kars-system","kars"); + const key="karssreregistrations.kars.azure.com//canonical"; + f.objects[key]={metadata:{name:"canonical",uid:"registration",resourceVersion:"7",generation:2}, + spec:{...spec,enabled:false,legacyConsumer:{namespace:"kars-sre",name:"sre",uid:"retired-consumer",resourceVersion:"2"}, + legacyBindings:[{name:"retired-grant"}]},status:{phase:"Retired",observedGeneration:2}}; + const execute=vi.fn(async(file,args,options)=>{ + if(args[0]!=="patch") return f.execute(file,args,options); + expect(args).toContain("--type=json"); + const operations=JSON.parse(args[args.indexOf("-p")+1]); + expect(operations).toEqual([ + {op:"test",path:"/metadata/uid",value:"registration"}, + {op:"test",path:"/metadata/resourceVersion",value:"7"}, + {op:"replace",path:"/spec",value:spec}, + ]); + f.objects[key].spec=structuredClone(operations[2].value); + return {stdout:""}; + }); + await enroll(execute,spec,{ + sandboxUid:"source",namespaceUid:"runtime-ns",binding:[], + registrationUid:"registration",resourceVersion:"7", + ...(hasConsumer?{consumer:"new-consumer@9"}:{}), + },false); + expect(f.objects[key].spec).toEqual(spec); + expect(f.objects[key].spec.legacyConsumer).toEqual(hasConsumer?spec.legacyConsumer:undefined); + expect(f.objects[key].metadata.uid).toBe("registration"); + }); + + it("does not patch re-enrollment against a stale registration identity",async()=>{ + const f=fixture(); + const spec=await preview(f.execute,"kars-system","kars"); + f.objects["karssreregistrations.kars.azure.com//canonical"]={ + metadata:{name:"canonical",uid:"replacement-registration",resourceVersion:"8",generation:1},spec}; + await expect(enroll(f.execute,spec,{ + sandboxUid:"source",namespaceUid:"runtime-ns",binding:[], + registrationUid:"registration",resourceVersion:"7", + },false)).rejects.toThrow("reviewed registration UID/resourceVersion"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="patch")).toBe(false); + }); + + it("refuses to stage over any preexisting source or namespace",async()=>{ + const f=fixture(); + await expect(stageSource(f.execute,"","kars-system","kars")).rejects.toThrow("already exists"); + delete f.objects["karssandbox/kars-system/sre"]; + await expect(stageSource(f.execute,"","kars-system","kars")).rejects.toThrow("Existing kars-sre"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="create"||args[0]==="patch")).toBe(false); + }); + + it("captures CREATE UID and never converts a racing 409 into adoption",async()=>{ + const f=fixture(); + delete f.objects["karssandbox/kars-system/sre"]; + delete f.objects["namespace//kars-sre"]; + const base=f.execute; + const execute=vi.fn(async(file,args,options)=>{ + if(args[0]==="create")throw new Error("409 AlreadyExists"); + return base(file,args,options); + }); + const rendered=JSON.stringify({apiVersion:"kars.azure.com/v1alpha1",kind:"KarsSandbox", + metadata:{name:"sre",namespace:"kars-system"},spec:{runtime:{kind:"Hermes"}}}); + await expect(stageSource(execute,rendered,"kars-system","kars")).rejects.toThrow("409"); + const createIndex=execute.mock.calls.findIndex(([,args])=>args[0]==="create"); + expect(createIndex).toBeGreaterThanOrEqual(0); + expect(execute.mock.calls.slice(createIndex+1)).toEqual([]); + }); + + it("enrolls the actual newly created source only after its namespace claim converges",async()=>{ + const f=fixture(); + const source=f.objects["karssandbox/kars-system/sre"]; + const runtime=f.objects["namespace//kars-sre"]; + delete f.objects["karssandbox/kars-system/sre"]; + delete f.objects["namespace//kars-sre"]; + const execute=vi.fn(async(file,args,options)=>{ + if(args[0]==="create"&&JSON.parse(options.input!).kind==="KarsSandbox") { + f.objects["karssandbox/kars-system/sre"]=source; + f.objects["namespace//kars-sre"]=runtime; + return {stdout:JSON.stringify(source)}; + } + return f.execute(file,args,options); + }); + const created=await stageSource(execute,JSON.stringify(source),"kars-system","kars"); + expect(created).toEqual({uid:"source",namespaceUid:"runtime-ns"}); + await enroll(execute,await preview(execute,"kars-system","kars"),{ + sandboxUid:created.uid,namespaceUid:created.namespaceUid,binding:[], + },false); + expect(f.objects["karssreregistrations.kars.azure.com//canonical"].spec.sandbox.uid).toBe("source"); + expect(execute.mock.calls.filter(([,args])=>args[0]==="create")).toHaveLength(2); + }); + + it("waits through controller migration without treating transient drain as failure",async()=>{ + vi.useFakeTimers(); + try { + const f=fixture(); + const key="karssreregistrations.kars.azure.com//canonical"; + f.objects[key]={metadata:{name:"canonical",uid:"registration",resourceVersion:"1",generation:1}, + status:{phase:"Migrating",observedGeneration:1,privacyRevision:"kars.azure.com/sre-privacy/v2"}}; + const pending=waitForAuthority(f.execute,"Ready",3); + setTimeout(()=>{f.objects[key].status.phase="Ready"},2100); + await vi.runAllTimersAsync(); + await pending; + } finally { vi.useRealTimers(); } + }); + + it.each([ + {version:"v3.16.4",dryRun:true}, {version:"v3.16.4",dryRun:false}, + {version:"v4.2.4",dryRun:true}, {version:"v4.2.4",dryRun:false}, + ])("stages an existing release with $version (dry-run: $dryRun)",async({version,dryRun})=>{ + const warn=vi.spyOn(console,"warn").mockImplementation(()=>{}); + try { + const f=fixture(); + f.objects["deployment/kars-system/kars-controller"].spec={ + template:{spec:{serviceAccountName:"kars-controller"}}, + }; + const execute=vi.fn(async(file,args,options)=>{ + if(file==="helm"&&args[0]==="list") { + if(version.startsWith("v4.")&&args.includes("--all")) { + throw Object.assign(new Error("Unsupported flag"),{ + exitCode:1,stderr:"Error: unknown flag: --all\n", + }); + } + return {stdout:'[{"name":"kars","namespace":"kars-system","status":"pending-upgrade"}]'}; + } + if(file==="helm"&&args[0]==="version")return {stdout:version}; + if(file==="helm"&&args[0]==="template")return {stdout:actionCrd}; + if(file==="helm"&&args[0]==="upgrade")return {stdout:""}; + return f.execute(file,args,options); + }); + await stageAuthority(execute,"chart","kars-system","kars","new/controller:latest","new/router:latest",dryRun); + const upgrade=execute.mock.calls.find(([file,args])=>file==="helm"&&args[0]==="upgrade"); + expect(upgrade?.[1]).toContain("--reset-then-reuse-values"); + expect(upgrade?.[1]).toContain("sre.authorityStage=true"); + expect(upgrade?.[1]).toContain(dryRun?"--dry-run=server":version.startsWith("v4.")?"--wait=legacy":"--wait"); + expect(execute.mock.calls.some(([,args])=>args[0]==="install")).toBe(false); + expect(execute.mock.calls.some(([,args])=>args[0]==="create")).toBe(!dryRun); + } finally { warn.mockRestore(); } + }); + + it("stages legacy template installations using owned resources and controller CAS without overwriting other env",async()=>{ + const f=fixture(); + const controller=f.objects["deployment/kars-system/kars-controller"]; + controller.spec={template:{spec:{serviceAccountName:"kars-controller",containers:[ + {name:"controller",image:"old/controller:latest",env:[{name:"PRESERVED",value:"setting"}]}, + ]}}}; + const execute=vi.fn(async(file,args,options)=>{ + if(file==="helm"&&args[0]==="list")return {stdout:"[]"}; + if(file==="helm")return {stdout:JSON.stringify({kind:"CustomResourceDefinition", + apiVersion:"apiextensions.k8s.io/v1",metadata:{name:"karssreregistrations.kars.azure.com"},spec:{scope:"Cluster"}})+"\n"+actionCrd}; + return f.execute(file,args,options); + }); + delete f.objects["crd//karssreregistrations.kars.azure.com"]; + await stageAuthority(execute,"chart","kars-system","kars","new/controller:latest","new/router:latest",false); + const patch=execute.mock.calls.find(([,args])=>args[0]==="patch")!; + const body=JSON.parse(patch[1][patch[1].indexOf("-p")+1]); + expect(body.metadata).toEqual({uid:"controller",resourceVersion:"1"}); + expect(body.spec.template.spec.containers[0].env).toContainEqual({name:"PRESERVED",value:"setting"}); + expect(body.spec.template.spec.containers[0].env).toContainEqual({name:"INFERENCE_ROUTER_IMAGE",value:"new/router:latest"}); + expect(execute.mock.calls.some(([,args])=>args.includes("--force-conflicts"))).toBe(false); + }); + + it.each([ + {apiGroups:[""],resources:["secrets"],verbs:["watch"]}, + {apiGroups:["*"],resources:["*"],verbs:["watch"]}, + {apiGroups:[""],resources:["secrets"],verbs:["*"]}, + {apiGroups:[""],resources:["pods","secrets"],verbs:["get","watch"]}, + ])("rejects group Secret watch authority before any mutation: %j",async rule=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"watcher",uid:"binding",resourceVersion:"1"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"watcher"}, + subjects:[{kind:"Group",name:"system:serviceaccounts:kars-sre",apiGroup:"rbac.authorization.k8s.io"}]}); + f.objects["clusterrole//watcher"]={metadata:{name:"watcher",uid:"role",resourceVersion:"1"},rules:[rule]}; + await expect(assertSafeMutation(f.execute)).rejects.toThrow("broad group grant"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + }); + + it("exempts only the exact ordinary non-Secret spawner role, not a same-name watch grant",async()=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"spawner",uid:"binding",resourceVersion:"1"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"kars-sandbox-spawner"}, + subjects:[{kind:"ServiceAccount",namespace:"kars-sre",name:"sandbox"}]}); + const role={metadata:{name:"kars-sandbox-spawner",uid:"role",resourceVersion:"1"}, + rules:[{apiGroups:["kars.azure.com"],resources:["karssandboxes"],verbs:["get","list","create","delete"]}]}; + f.objects["clusterrole//kars-sandbox-spawner"]=role; + await assertSafeMutation(f.execute); + role.rules=[{apiGroups:[""],resources:["secrets"],verbs:["watch"]}]; + await expect(assertSafeMutation(f.execute)).rejects.toThrow("Legacy SRE grants"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + }); +}); diff --git a/cli/src/lib/sre-authority.ts b/cli/src/lib/sre-authority.ts new file mode 100644 index 000000000..fedffa8b2 --- /dev/null +++ b/cli/src/lib/sre-authority.ts @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { namespaceClaimed, type OwnershipObject } from "./namespace-ownership.js"; + +export type Execute = (file: string, args: readonly string[], options: { stdio: "pipe"; input?: string }) => Promise<{ stdout: string }>; +const RESOURCE = "karssreregistrations.kars.azure.com"; +const CRD = "karssreregistrations.kars.azure.com"; +const NS = "kars-sre"; +const OLD_USER = `system:serviceaccount:${NS}:sandbox`; +const PRIVACY_REVISION = "kars.azure.com/sre-privacy/v2"; + +export interface ApiObject extends OwnershipObject { + metadata: OwnershipObject["metadata"] & { generation?: number }; + spec?: Record; + status?: Record; + roleRef?: { apiGroup: string; kind: string; name: string }; + subjects?: Array<{ kind: string; name: string; namespace?: string; apiGroup?: string }>; + rules?: Array<{ apiGroups?: string[]; resources?: string[]; verbs?: string[]; nonResourceURLs?: string[] }>; +} + +export interface Enrollment { + controller: { namespace: { name: string; uid: string }; deployment: { name: string; uid: string }; release: string }; + sandbox: { namespace: string; name: string; uid: string }; + runtimeNamespace: { name: string; uid: string }; + enabled: boolean; + legacyBindings: Array<{ + kind: string; namespace?: string; name: string; uid: string; resourceVersion: string; + roleRef: NonNullable; subjects: NonNullable; + }>; + legacyConsumer?: { namespace: string; name: string; uid: string; resourceVersion: string }; +} + +export async function get(execute: Execute, kind: string, name: string, namespace?: string): Promise { + const { stdout } = await execute("kubectl", ["get", kind, name, ...(namespace ? ["-n", namespace] : []), + "--ignore-not-found", "-o", "json"], { stdio: "pipe" }); + if (!stdout.trim()) return undefined; + const object = JSON.parse(stdout) as ApiObject; + if (!object.metadata?.uid || !object.metadata.resourceVersion || !object.metadata.name) { + throw new Error("SRE authority API response omitted its exact identity"); + } + return object; +} + +async function list(execute: Execute, kind: string): Promise { + const { stdout } = await execute("kubectl", ["get", kind, "-A", "-o", "json"], { stdio: "pipe" }); + const result = JSON.parse(stdout) as { items?: ApiObject[] }; + if (!Array.isArray(result.items)) throw new Error("SRE authority inventory is malformed"); + for (const item of result.items) if (!item.metadata?.uid || !item.metadata.resourceVersion || !item.metadata.name) { + throw new Error("SRE authority inventory omitted an object identity"); + } + return result.items; +} + +export function oldSubject(subject: NonNullable[number]): boolean { + return (subject.kind === "ServiceAccount" && subject.name === "sandbox" && subject.namespace === NS) + || (subject.kind === "User" && subject.name === OLD_USER); +} + +function oldGroup(subject: NonNullable[number]): boolean { + return subject.kind === "Group" + && ["system:authenticated", "system:serviceaccounts", `system:serviceaccounts:${NS}`].includes(subject.name); +} + +function dangerous(role: ApiObject): boolean { + const includes = (items: string[] | undefined, value: string) => items?.some(item => item === "*" || item === value); + return role.rules?.some(rule => + (includes(rule.apiGroups, "") && includes(rule.resources, "secrets") + && ["get", "list", "watch"].some(verb => includes(rule.verbs, verb))) + || (includes(rule.apiGroups, "") && ["pods/exec", "pods/proxy", "serviceaccounts/token"] + .some(resource => includes(rule.resources, resource)) + && (includes(rule.verbs, "get") || includes(rule.verbs, "create"))) + || (includes(rule.apiGroups, "kars.azure.com") && includes(rule.resources, "karssreactions") && includes(rule.verbs, "create")), + ) ?? false; +} + +export async function legacyBindings(execute: Execute): Promise { + const bindings = [...await list(execute, "clusterrolebindings"), ...await list(execute, "rolebindings")]; + const result: ApiObject[] = []; + for (const binding of bindings) { + if (!binding.roleRef) throw new Error("RBAC binding omitted roleRef"); + const subjects = binding.subjects ?? []; + if (subjects.some(oldGroup)) { + const role = await get(execute, binding.roleRef.kind.toLowerCase(), binding.roleRef.name, + binding.roleRef.kind === "Role" ? binding.metadata.namespace : undefined); + if (!role) throw new Error("A referenced RBAC role is missing"); + if (dangerous(role)) throw new Error("A broad group grant gives the legacy SRE identity privileged access; restructure it explicitly before migration"); + } + if (subjects.some(oldSubject)) { + let ordinarySpawner = false; + if (binding.roleRef.kind === "ClusterRole" && binding.roleRef.name === "kars-sandbox-spawner") { + const role = await get(execute, "clusterrole", binding.roleRef.name); + if (!role) throw new Error("A referenced spawner role is missing"); + ordinarySpawner = (role.rules ?? []).every(rule => + JSON.stringify(rule.apiGroups) === '["kars.azure.com"]' + && JSON.stringify(rule.resources) === '["karssandboxes"]' + && !rule.nonResourceURLs + && (rule.verbs ?? []).every(verb => ["get", "list", "create", "delete"].includes(verb))); + } + if (!ordinarySpawner) result.push(binding); + } + } + return result; +} + +export async function requireRegistrar(execute: Execute): Promise { + const { stdout } = await execute("kubectl", ["auth", "can-i", "use", `${RESOURCE}/canonical`], { stdio: "pipe" }); + if (stdout.trim() !== "yes") throw new Error("Explicit cluster-level kars-sre-registrar permission is required"); +} + +export async function preview(execute: Execute, namespace: string, release: string): Promise { + const controllerNs = await get(execute, "namespace", namespace); + const controller = await get(execute, "deployment", "kars-controller", namespace); + const sandbox = await get(execute, "karssandbox", "sre", namespace); + const runtime = await get(execute, "namespace", NS); + if (!controllerNs || !controller || !sandbox || !runtime) throw new Error("Stage the unprivileged canonical SRE source and wait for its namespace claim before enrollment"); + if (controllerNs.metadata.deletionTimestamp || controller.metadata.deletionTimestamp + || sandbox.metadata.deletionTimestamp || runtime.metadata.deletionTimestamp + || !namespaceClaimed(runtime, sandbox) + || sandbox.metadata.annotations?.["kars.azure.com/namespace-uid"] !== runtime.metadata.uid) { + throw new Error("Canonical SRE source/runtime identity or claim is not live and complete"); + } + const annotations = controller.metadata.annotations ?? {}; + if ((annotations["meta.helm.sh/release-name"] && annotations["meta.helm.sh/release-name"] !== release) + || (annotations["meta.helm.sh/release-namespace"] && annotations["meta.helm.sh/release-namespace"] !== namespace)) { + throw new Error("Controller belongs to another release"); + } + const bindings = await legacyBindings(execute); + const consumer = await get(execute, "deployment", "sre", NS); + return { + controller: { + namespace: { name: namespace, uid: controllerNs.metadata.uid! }, + deployment: { name: "kars-controller", uid: controller.metadata.uid! }, release, + }, + sandbox: { namespace, name: "sre", uid: sandbox.metadata.uid! }, + runtimeNamespace: { name: NS, uid: runtime.metadata.uid! }, + enabled: true, + legacyBindings: bindings.map(binding => ({ + kind: binding.kind!, ...(binding.metadata.namespace ? { namespace: binding.metadata.namespace } : {}), + name: binding.metadata.name!, uid: binding.metadata.uid!, resourceVersion: binding.metadata.resourceVersion!, + roleRef: binding.roleRef!, subjects: binding.subjects ?? [], + })), + ...(consumer ? { legacyConsumer: { namespace: NS, name: "sre", uid: consumer.metadata.uid!, + resourceVersion: consumer.metadata.resourceVersion! } } : {}), + }; +} + +export interface Review { + sandboxUid: string; + namespaceUid: string; + binding: string[]; + consumer?: string; + registrationUid?: string; + resourceVersion?: string; +} + +export async function enroll(execute: Execute, spec: Enrollment, review: Review, dryRun: boolean): Promise { + await requireRegistrar(execute); + if (spec.sandbox.uid !== review.sandboxUid || spec.runtimeNamespace.uid !== review.namespaceUid) { + throw new Error("Reviewed SRE source/namespace UID no longer matches"); + } + const expected = spec.legacyBindings.map(binding => + `${binding.kind}/${binding.namespace ?? ""}/${binding.name}=${binding.uid}@${binding.resourceVersion}`); + if (expected.length !== review.binding.length || expected.some(value => !review.binding.includes(value))) { + throw new Error(`Every legacy binding must be explicitly reviewed: ${expected.join(", ")}`); + } + const consumer = spec.legacyConsumer; + if (consumer && review.consumer !== `${consumer.uid}@${consumer.resourceVersion}`) { + throw new Error("Existing SRE consumer requires an exact --consumer UID@resourceVersion review"); + } + const object = { apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSRERegistration", + metadata: { name: "canonical" }, spec }; + const existing = await get(execute, RESOURCE, "canonical"); + if (dryRun) return JSON.stringify(object, null, 2); + if (existing) { + if (existing.metadata.uid !== review.registrationUid || existing.metadata.resourceVersion !== review.resourceVersion) { + throw new Error("Updating an enrollment requires its reviewed registration UID/resourceVersion"); + } + // Enrollment is the complete reviewed spec, not an overlay on retired data. + await execute("kubectl", ["patch", RESOURCE, "canonical", "--type=json", "-p", JSON.stringify([ + { op: "test", path: "/metadata/uid", value: review.registrationUid }, + { op: "test", path: "/metadata/resourceVersion", value: review.resourceVersion }, + { op: "replace", path: "/spec", value: spec }, + ])], { stdio: "pipe" }); + } else { + if (review.registrationUid || review.resourceVersion) throw new Error("Reviewed registration disappeared"); + await execute("kubectl", ["create", "-f", "-", "-o", "json"], { stdio: "pipe", input: JSON.stringify(object) }); + } + return "SRE enrollment recorded; the controller will perform the reviewed migration"; +} + +export async function registration(execute: Execute): Promise { + if (!await get(execute, "crd", CRD)) return undefined; + return get(execute, RESOURCE, "canonical"); +} + +/** Read-only normal-deployment gate. Explicit authority staging is a separate + * registrar operation; ordinary deploy paths must never silently retire grants. */ +export async function assertSafeMutation(execute: Execute): Promise { + const reg = await registration(execute); + if (!reg && !await get(execute, "namespace", NS)) return; + const bindings = await legacyBindings(execute); + if (bindings.length) throw new Error("Legacy SRE grants require 'kars sre authority stage/preview/enroll' before deployment changes"); + if (!reg) return; + if (reg.spec?.enabled === false && reg.status?.phase === "Retired" + && reg.status.observedGeneration === reg.metadata.generation) return; + const spec = reg.spec as unknown as Enrollment; + const current = await preview(execute, spec.controller.namespace.name, spec.controller.release); + if (current.sandbox.uid !== spec.sandbox.uid || current.runtimeNamespace.uid !== spec.runtimeNamespace.uid + || current.controller.deployment.uid !== spec.controller.deployment.uid + || current.controller.namespace.uid !== spec.controller.namespace.uid + || reg.status?.observedGeneration !== reg.metadata.generation + || reg.status?.privacyRevision !== PRIVACY_REVISION + || !["Ready", "Retired"].includes(reg.status?.phase ?? "")) { + throw new Error("SRE authority is stale or migration is incomplete; deployment changes stopped"); + } +} + +export async function assertRollbackSafe(execute: Execute): Promise { + if (await registration(execute)) { + throw new Error("Rollback across enrolled SRE authority is unsafe; use a reviewed roll-forward release instead"); + } + await assertSafeMutation(execute); +} + +export async function waitForAuthority(execute: Execute, phase: "Ready" | "Retired", attempts = 90): Promise { + for (let attempt = 0; attempt < attempts; attempt++) { + const reg = await registration(execute); + if (reg?.status?.phase === phase && reg.status.observedGeneration === reg.metadata.generation + && reg.status.privacyRevision === PRIVACY_REVISION) return; + if (reg?.status?.phase === "Blocked" && reg.status.observedGeneration === reg.metadata.generation) { + throw new Error(`SRE migration blocked: ${String(reg.status.detail ?? "inspect registration status")}`); + } + await new Promise(resolve => setTimeout(resolve, 2000)); + } + throw new Error("SRE authority did not converge; verify the prerequisite controller/router images and registration status"); +} + +export async function retire(execute: Execute, uid: string, resourceVersion: string): Promise { + await requireRegistrar(execute); + const reg = await registration(execute); + if (!reg || reg.metadata.uid !== uid || reg.metadata.resourceVersion !== resourceVersion) { + throw new Error("Reviewed registration UID/resourceVersion changed"); + } + await execute("kubectl", ["patch", RESOURCE, "canonical", "--type=merge", "-p", JSON.stringify({ + metadata: { uid, resourceVersion }, spec: { enabled: false }, + })], { stdio: "pipe" }); + await waitForAuthority(execute, "Retired"); +} + +export async function assertDestroySafe(execute: Execute): Promise { + const reg = await registration(execute); + if (!reg && !await get(execute, "namespace", NS)) return; + if (reg && (reg.status?.phase !== "Retired" || reg.spec?.enabled !== false + || reg.status.observedGeneration !== reg.metadata.generation)) { + throw new Error("Retire the reviewed SRE registration before uninstalling or destroying its controller"); + } + if ((await legacyBindings(execute)).length) throw new Error("Unretired legacy SRE bindings block uninstall/destroy"); +} diff --git a/cli/src/lib/sre-helm.test.ts b/cli/src/lib/sre-helm.test.ts new file mode 100644 index 000000000..b5336cf6b --- /dev/null +++ b/cli/src/lib/sre-helm.test.ts @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { execa } from "execa"; +import type { Execute } from "./sre-authority.js"; +import { listSreHelmReleases, sreHelmStageWait } from "./sre-helm.js"; + +const inventory = JSON.stringify([{ name: "kars", namespace: "workspace", status: "pending-upgrade" }]); +const unsupported = () => Object.assign(new Error("Helm flag rejected"), { + exitCode: 1, stderr: "Error: unknown flag: --all\n", +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("SRE Helm release inventory compatibility", () => { + it("handles the installed Helm flag parser without contacting a cluster", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + // Help keeps listing offline, but still rejects unsupported flags. + const execute: Execute = (file, args, options) => + execa(file, [...args, ...(args[0] === "list" ? ["--help"] : [])], options); + const output = await listSreHelmReleases(execute, "workspace"); + expect(output).toContain("helm list"); + expect(output).toContain("--all-namespaces"); + }); + + it("retains Helm 3 all-status discovery without an extra probe", async () => { + const execute = vi.fn().mockResolvedValue({ stdout: inventory }); + expect(await listSreHelmReleases(execute, "workspace")).toBe(inventory); + expect(execute.mock.calls).toEqual([ + ["helm", ["list", "-n", "workspace", "--all", "-o", "json"], { stdio: "pipe" }], + ]); + }); + + it.each(["v4.0.0", "v4.2.4", "v4.1.0-rc.1+build.2"])( + "uses confirmed Helm %s all-status defaults and preserves executor context", async version => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const execute = vi.fn() + .mockRejectedValueOnce(unsupported()) + .mockResolvedValueOnce({ stdout: version }) + .mockResolvedValueOnce({ stdout: inventory }); + const contextual: Execute = (file, args, options) => + execute(file, ["--kube-context", "test-context", ...args], options); + expect(await listSreHelmReleases(contextual, "workspace")).toBe(inventory); + expect(execute.mock.calls.map(([, args]) => args)).toEqual([ + ["--kube-context", "test-context", "list", "-n", "workspace", "--all", "-o", "json"], + ["--kube-context", "test-context", "version", "--template", "{{.Version}}"], + ["--kube-context", "test-context", "list", "-n", "workspace", "-o", "json"], + ]); + }, + ); + + it.each([ + { exitCode: 1, stderr: "Forbidden: release storage access denied" }, + { exitCode: 1, stderr: "Error: unknown flag: --all-namespaces" }, + { exitCode: 1, stderr: "Error: unknown flag: --all\nanother failure" }, + { exitCode: 2, stderr: "Error: unknown flag: --all" }, + ])("does not retry another discovery failure: %j", async detail => { + const error = Object.assign(new Error("Discovery failed"), detail); + const execute = vi.fn().mockRejectedValue(error); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toBe(error); + expect(execute).toHaveBeenCalledTimes(1); + }); + + it.each(["v3.19.0", "v5.0.0", "", "unexpected output"])( + "does not assume default all-status semantics for %j", async version => { + const execute = vi.fn().mockRejectedValueOnce(unsupported()) + .mockResolvedValueOnce({ stdout: version }); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toThrow(/only Helm 4|Unsupported Helm version/); + expect(execute).toHaveBeenCalledTimes(2); + }, + ); + + it("propagates a failed Helm 4 retry rather than reporting an absent release", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const denied = new Error("Forbidden"); + const execute = vi.fn().mockRejectedValueOnce(unsupported()) + .mockResolvedValueOnce({ stdout: "v4.2.4" }).mockRejectedValueOnce(denied); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toBe(denied); + expect(execute).toHaveBeenCalledTimes(3); + }); + + describe("SRE staging waits only for built-ins before explicit enrollment", () => { + it.each([["v3.16.4", "--wait"], ["v4.2.4", "--wait=legacy"], ["v4.0.0-rc.1", "--wait=legacy"]])( + "selects %s's bounded built-in waiter", async (version, expected) => { + const execute = vi.fn().mockResolvedValue({ stdout: version }); + expect(await sreHelmStageWait(execute)).toBe(expected); + expect(execute.mock.calls).toEqual([ + ["helm", ["version", "--template", "{{.Version}}"], { stdio: "pipe" }], + ]); + }, + ); + it.each(["", "v5.0.0", "v4.2", "v3.16.4\nwarning", "unknown"])("fails closed for %j", async version => { + await expect(sreHelmStageWait(vi.fn().mockResolvedValue({ stdout: version }))) + .rejects.toThrow("Unsupported Helm"); + }); + it("does not replace version errors with a default waiter", async () => { + const error = new Error("Helm unavailable"); + await expect(sreHelmStageWait(vi.fn().mockRejectedValue(error))).rejects.toBe(error); + }); + it("passes the real installed Helm upgrade parser without a Kubernetes connection", async () => { + const wait = await sreHelmStageWait(execa); + const { stdout } = await execa("helm", ["upgrade", "kars", "chart", wait, "--timeout", "8m", "--help"], { stdio: "pipe" }); + expect(stdout).toContain("helm upgrade"); + }); + }); +}); diff --git a/cli/src/lib/sre-helm.ts b/cli/src/lib/sre-helm.ts new file mode 100644 index 000000000..d83e5a2ce --- /dev/null +++ b/cli/src/lib/sre-helm.ts @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Execute } from "./sre-authority.js"; + +export async function sreHelmMajor(execute: Execute): Promise<3 | 4> { + const { stdout } = await execute("helm", ["version", "--template", "{{.Version}}"], { stdio: "pipe" }); + const match = /^v([34])\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?$/.exec(stdout.trim()); + if (!match) throw new Error("Unsupported Helm version; SRE staging requires Helm 3 or 4."); + return Number(match[1]) as 3 | 4; +} + +export async function sreHelmStageWait(execute: Execute): Promise { + // The Helm 4 watcher waits for custom-resource Ready, which requires the + // explicit enrollment AFTER staging. Its legacy waiter checks built-ins/APIs. + return await sreHelmMajor(execute) === 4 ? "--wait=legacy" : "--wait"; +} + +function removedAllFlag(error: unknown): boolean { + return error instanceof Error + && "exitCode" in error && error.exitCode === 1 + && "stderr" in error && typeof error.stderr === "string" + && error.stderr.trim() === "Error: unknown flag: --all"; +} + +export async function listSreHelmReleases(execute: Execute, namespace: string): Promise { + try { + return (await execute("helm", ["list", "-n", namespace, "--all", "-o", "json"], { stdio: "pipe" })).stdout; + } catch (error) { + if (!removedAllFlag(error)) throw error; + if (await sreHelmMajor(execute) !== 4) { + throw new Error("Unsupported Helm release inventory: only Helm 4 can omit the --all flag.", { cause: error }); + } + // Helm 3 needs --all; Helm 4 removed it and lists every status by default. + console.warn("Helm 4 lists all release statuses by default; retrying without the removed --all flag."); + return (await execute("helm", ["list", "-n", namespace, "-o", "json"], { stdio: "pipe" })).stdout; + } +} diff --git a/cli/src/lib/sre-source.ts b/cli/src/lib/sre-source.ts new file mode 100644 index 000000000..88bb3a578 --- /dev/null +++ b/cli/src/lib/sre-source.ts @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { parseAllDocuments } from "yaml"; +import { namespaceClaimed } from "./namespace-ownership.js"; +import { get, legacyBindings, requireRegistrar, type ApiObject, type Execute } from "./sre-authority.js"; + +function same(left: unknown, right: unknown): boolean { + const canonical = (value: any): any => Array.isArray(value) ? value.map(canonical) + : value && typeof value === "object" ? Object.fromEntries(Object.keys(value).sort().map(key => [key,canonical(value[key])])) : value; + return JSON.stringify(canonical(left)) === JSON.stringify(canonical(right)); +} + +/** Stage only unprivileged configuration and CREATE a genuinely new source. + * A 409 never becomes a get/adopt of a racing tenant-created Sandbox. */ +export async function stageSource( + execute: Execute, rendered: string, namespace: string, release: string, +): Promise<{ uid: string; namespaceUid: string }> { + await requireRegistrar(execute); + if ((await legacyBindings(execute)).length) throw new Error("Existing SRE grants require explicit authority preview/enrollment before staging"); + if (await get(execute,"karssandbox","sre",namespace)) { + throw new Error("SRE source already exists; preview and enroll its explicitly reviewed UID instead of adopting by name"); + } + if (await get(execute,"namespace","kars-sre")) { + throw new Error("Existing kars-sre namespace requires explicit ownership review; foreign or unfinished occupants are preserved"); + } + const objects = parseAllDocuments(rendered).map(document => document.toJSON() as ApiObject | null).filter((value): value is ApiObject => !!value); + const source = objects.find(value => value.kind === "KarsSandbox" && value.metadata.name === "sre"); + if (!source || source.metadata.namespace !== namespace) throw new Error("Rendered canonical SRE source is absent or has the wrong workspace"); + const allowed = new Map([ + ["InferencePolicy","sre-inference"], ["ToolPolicy","sre-tools"], + ]); + const supports = objects.filter(value => + allowed.get(value.kind!) === value.metadata.name + || (value.kind === "ClusterRole" && ["kars-sre-reader","kars-sre-action-author","kars-sre-approver"].includes(value.metadata.name!))); + const missing: ApiObject[] = []; + // Validate the entire support set before writing any of it. + for (const object of supports) { + const existing = await get(execute,object.kind!.toLowerCase(),object.metadata.name!,object.metadata.namespace); + if (existing) { + if (!same(existing.spec ?? existing.rules,object.spec ?? object.rules)) { + throw new Error(`Existing ${object.kind}/${object.metadata.name} differs; no support resource was adopted or overwritten`); + } + } else missing.push(object); + } + const own = (object: ApiObject) => ({ + ...object, metadata: { + ...object.metadata, + labels: { ...object.metadata.labels, "app.kubernetes.io/managed-by":"Helm" }, + annotations: { ...object.metadata.annotations, + "meta.helm.sh/release-name":release,"meta.helm.sh/release-namespace":namespace }, + }, + }); + for (const object of missing) { + await execute("kubectl",["create","-f","-"],{stdio:"pipe",input:JSON.stringify(own(object))}); + } + const {stdout}=await execute("kubectl",["create","-f","-","-o","json"],{stdio:"pipe",input:JSON.stringify(own(source))}); + const created=JSON.parse(stdout) as ApiObject; + if (created.metadata?.namespace!==namespace || created.metadata.name!=="sre" || !created.metadata.uid) { + throw new Error("SRE CREATE response omitted its actual source UID/workspace"); + } + for (let attempt=0;attempt<60;attempt++) { + const live=await get(execute,"karssandbox","sre",namespace); + if (!live || live.metadata.uid!==created.metadata.uid || live.metadata.deletionTimestamp) { + throw new Error("Created SRE source was replaced or deleted during namespace staging"); + } + const runtime=await get(execute,"namespace","kars-sre"); + if (runtime) { + if (runtime.metadata.deletionTimestamp || !namespaceClaimed(runtime,live)) { + throw new Error("SRE runtime namespace belongs to a foreign/ambiguous occupant; preserved without any grant"); + } + if (live.metadata.annotations?.["kars.azure.com/namespace-uid"]===runtime.metadata.uid) { + return {uid:created.metadata.uid,namespaceUid:runtime.metadata.uid!}; + } + } + await new Promise(resolve=>setTimeout(resolve,1000)); + } + throw new Error("SRE source was staged but its exact namespace claim did not converge"); +} diff --git a/cli/src/lib/sre-stage.test.ts b/cli/src/lib/sre-stage.test.ts new file mode 100644 index 000000000..e2890070d --- /dev/null +++ b/cli/src/lib/sre-stage.test.ts @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { execa } from "execa"; +import { parse } from "yaml"; +import { describe, expect, it, vi } from "vitest"; +import { ACTION_CRD, planActionCrd } from "./sre-action-crd.js"; +import { stageAuthority } from "./sre-stage.js"; +import type { Execute } from "./sre-authority.js"; + +const action = parse(readFileSync(new URL("../../../deploy/helm/kars/templates/crd-karssreaction.yaml", import.meta.url), "utf8")); +const registration = { apiVersion: "apiextensions.k8s.io/v1", kind: "CustomResourceDefinition", + metadata: { name: "karssreregistrations.kars.azure.com" }, spec: { scope: "Cluster" } }; +const policy = { apiVersion: "admissionregistration.k8s.io/v1", kind: "ValidatingAdmissionPolicy", + metadata: { name: "kars-sre-test" }, spec: { failurePolicy: "Fail" } }; +const params = (object: any) => object.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.action.properties.params; + +function fixture(helm = false) { + const existing = structuredClone(action); + existing.metadata.uid = "action-uid"; + existing.metadata.resourceVersion = "17"; + existing.metadata.annotations = { "operator.example/keep": "custom metadata" }; + if (helm) { + existing.metadata.labels["app.kubernetes.io/managed-by"] = "Helm"; + Object.assign(existing.metadata.annotations, { "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": "kars-system" }); + } + delete params(existing)["x-kubernetes-preserve-unknown-fields"]; + params(existing).additionalProperties = true; + const controller = { metadata: { name: "kars-controller", uid: "controller-uid", resourceVersion: "2" }, + spec: { template: { spec: { serviceAccountName: "kars-controller", containers: [{ name: "controller", image: "old:latest" }] } } } }; + const execute = vi.fn(async (file, args, options) => { + if (file === "helm") { + if (args[0] === "list") return { stdout: helm ? '[{"name":"kars","namespace":"kars-system"}]' : "[]" }; + if (args[0] === "version") return { stdout: "v4.2.4" }; + if (args[0] === "template") return { stdout: [action, registration, policy].map(obj => JSON.stringify(obj)).join("\n---\n") }; + if (args[0] === "upgrade") return { stdout: "" }; + } + if (args[0] === "auth") return { stdout: "yes" }; + if (args[0] === "get") return { stdout: args[2] === ACTION_CRD ? JSON.stringify(existing) + : args[1] === "deployment" ? JSON.stringify(controller) : "" }; + if (args[0] === "patch" && args[2] === ACTION_CRD) { + const operations = JSON.parse(args[args.indexOf("-p") + 1]); + for (const operation of operations) { + const segments = operation.path.slice(1).split("/"); + const target = segments.slice(0, -1).reduce((obj: any, key: string) => obj[key], existing); + const key = segments.at(-1); + if (operation.op === "test" && JSON.stringify(target[key]) !== JSON.stringify(operation.value)) throw new Error("409 UID/RV conflict"); + if (operation.op === "remove") delete target[key]; + if (operation.op === "add") target[key] = operation.value; + } + } + if (args[0] === "create") JSON.parse(options.input!); + return { stdout: "" }; + }); + const run = (dry = false, exec: Execute = execute) => stageAuthority(exec, "chart", "kars-system", "kars", "controller:latest", "router:latest", dry); + return { existing, controller, execute, run }; +} + +describe("existing action API prerequisite compatibility", () => { + it("feeds the actual offline Helm render through template staging with the repaired action API first", async () => { + const f = fixture(); + const execute: Execute = (file, args, options) => file === "helm" && args[0] === "template" + ? execa(file, args, options) : f.execute(file, args, options); + await stageAuthority(execute, fileURLToPath(new URL("../../../deploy/helm/kars", import.meta.url)), + "kars-system", "kars", "controller:latest", "router:latest", false); + expect(params(f.existing)["x-kubernetes-preserve-unknown-fields"]).toBe(true); + const created = f.execute.mock.calls.filter(([, args]) => args[0] === "create").map(([, , options]) => JSON.parse(options.input!)); + expect(created.some(obj => obj.metadata.name === registration.metadata.name)).toBe(true); + expect(created.filter(obj => obj.kind === "ValidatingAdmissionPolicy")).toHaveLength(14); + }); + + it.each([false, true])("repairs ONLY recognized legacy params before dependent policies (Helm: %s)", async helm => { + const f = fixture(helm); + const before = structuredClone(f.existing); + await f.run(); + const repaired = structuredClone(before); + delete params(repaired).additionalProperties; + params(repaired)["x-kubernetes-preserve-unknown-fields"] = true; + expect(f.existing).toEqual(repaired); + const calls = f.execute.mock.calls; + const patch = calls.findIndex(([, args]) => args[0] === "patch" && args[2] === ACTION_CRD); + const wait = calls.findIndex(([, args]) => args[0] === "wait" && args.includes(`crd/${ACTION_CRD}`)); + const dependent = calls.findIndex(([file, args, options]) => helm ? file === "helm" && args[0] === "upgrade" + : args[0] === "create" && JSON.parse(options.input!).kind === "ValidatingAdmissionPolicy"); + expect(patch).toBeGreaterThan(0); + expect(wait).toBeGreaterThan(patch); + expect(dependent).toBeGreaterThan(wait); + const operations = JSON.parse(calls[patch][1].at(-1)!); + expect(operations.slice(0, 2)).toEqual([ + { op: "test", path: "/metadata/uid", value: "action-uid" }, + { op: "test", path: "/metadata/resourceVersion", value: "17" }, + ]); + expect(f.existing.metadata.annotations["kars.azure.com/sre-authority-staged"]).toBeUndefined(); + if (helm) expect(calls[dependent][1]).toEqual(expect.arrayContaining(["--wait=legacy", "--timeout", "8m"])); + }); + + it("recognizes only Kubernetes API defaults while preserving their serialized fields", async () => { + const f = fixture(); + f.existing.spec.names.listKind = "KarsSREActionList"; + delete f.existing.spec.names.categories; + f.existing.spec.conversion = { strategy: "None" }; + f.existing.spec.preserveUnknownFields = false; + f.existing.spec.versions[0].deprecated = false; + f.existing.spec.versions[0].additionalPrinterColumns[0].priority = 0; + await f.run(); + expect(f.existing.spec.conversion).toEqual({ strategy: "None" }); + expect(f.existing.spec.names.listKind).toBe("KarsSREActionList"); + expect(f.existing.spec.names.categories).toBeUndefined(); + }); + + it.each([ + (obj: any) => { obj.metadata.annotations["meta.helm.sh/release-name"] = "foreign"; }, + (obj: any) => { obj.metadata.annotations["meta.helm.sh/release-namespace"] = "foreign"; }, + (obj: any) => { obj.metadata.annotations["kars.azure.com/sre-authority-staged"] = "foreign"; }, + (obj: any) => { obj.metadata.annotations["kars.azure.com/sre-authority-release"] = "foreign"; }, + (obj: any) => { obj.metadata.labels["app.kubernetes.io/managed-by"] = "other-operator"; }, + (obj: any) => { obj.metadata.ownerReferences = [{ uid: "foreign-owner" }]; }, + (obj: any) => { obj.metadata.deletionTimestamp = "2026-09-10T00:00:00Z"; }, + (obj: any) => { params(obj).properties = { custom: { type: "string" } }; }, + (obj: any) => { obj.spec.versions[0].schema.openAPIV3Schema.properties.spec.required.push("diagnosis"); }, + (obj: any) => { obj.spec.versions.push(structuredClone(obj.spec.versions[0])); }, + (obj: any) => { obj.spec.conversion = { strategy: "Webhook" }; }, + ])("refuses foreign/customized schemas without writing any authority resource: %#", async modify => { + const f = fixture(); + modify(f.existing); + await expect(f.run()).rejects.toThrow(/Foreign|Customized/); + expect(f.execute.mock.calls.some(([, args]) => ["patch", "create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it.each(["uid", "resourceVersion"])("rejects a racing %s replacement before policy creation", async field => { + const f = fixture(); + const stage = await planActionCrd(f.execute, action, "kars-system", "kars", false); + f.existing.metadata[field] = "replaced"; + await expect(stage()).rejects.toThrow("409"); + expect(params(f.existing).additionalProperties).toBe(true); + }); + + it.each([false, true])("propagates prerequisite failure before policies or Helm upgrade (Helm: %s)", async helm => { + const f = fixture(helm); + const execute = vi.fn(async (file, args, options) => { + if (args[0] === "wait" && args.includes(`crd/${ACTION_CRD}`)) throw new Error("Established timeout"); + return f.execute(file, args, options); + }); + + await expect(f.run(false, execute)).rejects.toThrow("Established timeout"); + expect(execute.mock.calls.some(([, args]) => ["create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it.each([false, true])("never continues after a forbidden prerequisite PATCH (Helm: %s)", async helm => { + const f = fixture(helm); + const execute: Execute = (file, args, options) => args[0] === "patch" && args[2] === ACTION_CRD + ? Promise.reject(new Error("Forbidden action API update")) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("Forbidden action API update"); + expect(f.execute.mock.calls.some(([, args]) => ["create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it("requires API establishment even when an existing registration schema is unchanged", async () => { + const f = fixture(); + const execute = vi.fn(async (file, args, options) => { + if (args[0] === "get" && args[2] === registration.metadata.name) return { stdout: JSON.stringify({ + ...registration, metadata: { ...registration.metadata, uid: "registration-api", resourceVersion: "1" }, + }) }; + if (args[0] === "wait" && args.includes(`crd/${registration.metadata.name}`)) throw new Error("Registration API not established"); + return f.execute(file, args, options); + }); + await expect(f.run(false, execute)).rejects.toThrow("Registration API not established"); + expect(execute.mock.calls.some(([, args]) => args[0] === "create")).toBe(false); + }); + + it("performs all template ownership preflights before repairing the prerequisite", async () => { + const f = fixture(); + const execute: Execute = (file, args, options) => args[0] === "get" && args[2] === policy.metadata.name + ? Promise.resolve({ stdout: JSON.stringify({ ...policy, metadata: { ...policy.metadata, uid: "foreign", resourceVersion: "1" }, + spec: { failurePolicy: "Ignore" } }) }) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("Unowned"); + expect(f.execute.mock.calls.some(([, args]) => args[0] === "patch")).toBe(false); + }); + + it.each([false, true])("dry-run never repairs APIs or mutates the controller (Helm: %s)", async helm => { + const f = fixture(helm); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + await f.run(true); + expect(params(f.existing).additionalProperties).toBe(true); + expect(f.execute.mock.calls.some(([, args]) => ["patch", "create", "wait", "rollout"].includes(args[0]))).toBe(false); + } finally { log.mockRestore(); } + }); + + it.each(["v5.0.0", "invalid"])("rejects unsupported Helm %s before any prerequisite write", async version => { + const f = fixture(true); + const execute: Execute = (file, args, options) => args[0] === "version" + ? Promise.resolve({ stdout: version }) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("Unsupported Helm"); + expect(f.execute.mock.calls.some(([, args]) => ["patch", "create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it("retains fatal rollout failures instead of accepting an unready controller", async () => { + const f = fixture(); + const execute: Execute = (file, args, options) => args[0] === "rollout" + ? Promise.reject(new Error("controller rollout failed")) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("controller rollout failed"); + }); +}); diff --git a/cli/src/lib/sre-stage.ts b/cli/src/lib/sre-stage.ts new file mode 100644 index 000000000..c460b4820 --- /dev/null +++ b/cli/src/lib/sre-stage.ts @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { parseAllDocuments } from "yaml"; +import { get, requireRegistrar, type ApiObject, type Execute } from "./sre-authority.js"; +import { listSreHelmReleases, sreHelmStageWait } from "./sre-helm.js"; +import { ACTION_CRD, planActionCrd } from "./sre-action-crd.js"; + +function parts(image: string): [string,string] { + const index=image.lastIndexOf(":"); + if(index<=image.lastIndexOf("/")||image.includes("@"))throw new Error("Stage images require explicit repository:tag references"); + return [image.slice(0,index),image.slice(index+1)]; +} + +function canonical(value:any):string { + const sort=(value:any):any=>Array.isArray(value)?value.map(sort) + :value&&typeof value==="object"?Object.fromEntries(Object.keys(value).sort().map(key=>[key,sort(value[key])])):value; + return JSON.stringify(sort(value)); +} + +export async function stageAuthority( + execute:Execute,chart:string,namespace:string,release:string, + controllerImage:string,routerImage:string,dryRun:boolean, +):Promise { + await requireRegistrar(execute); + const controller=await get(execute,"deployment","kars-controller",namespace); + if(!controller)throw new Error("Install the core prerequisite first; authority staging does not provision a new cluster"); + if(controller.spec?.template?.spec?.serviceAccountName!=="kars-controller") { + throw new Error("Controller uses a custom ServiceAccount; review and stage its minimal authority role explicitly"); + } + const stdout=await listSreHelmReleases(execute,namespace); + const releases=JSON.parse(stdout) as unknown; + if(!Array.isArray(releases)||releases.some(item=>!item||typeof item.name!=="string"||item.namespace!==namespace)) { + throw new Error("Helm ownership inventory is invalid"); + } + const [controllerRepository,controllerTag]=parts(controllerImage); + const [routerRepository,routerTag]=parts(routerImage); + const helm=releases.some(item=>item.name===release); + const rendered=await execute("helm",["template",release,chart,"--namespace",namespace, + "--set","sre.enabled=false","--set","azure.workloadIdentity.clientId=dummy"],{stdio:"pipe"}); + const documents=parseAllDocuments(rendered.stdout).map(doc=>{ + if(doc.errors.length)throw new Error(`Invalid staged chart YAML: ${doc.errors[0].message}`); + return doc.toJSON() as ApiObject|null; + }).filter((obj):obj is ApiObject=>!!obj); + const actions=documents.filter(obj=>obj.kind==="CustomResourceDefinition"&&obj.metadata.name===ACTION_CRD); + if(actions.length!==1)throw new Error("Exactly one compatible action CRD is required before staging authority policies"); + const stageAction=await planActionCrd(execute,actions[0],namespace,release,helm); + if(helm) { + const wait=await sreHelmStageWait(execute); + if(!dryRun)await stageAction(); + await execute("helm",["upgrade",release,chart,"--namespace",namespace,"--reset-then-reuse-values", + "--set","sre.authorityStage=true", + "--set-string",`controller.image.repository=${controllerRepository}`, + "--set-string",`controller.image.tag=${controllerTag}`, + "--set-string",`inferenceRouter.image.repository=${routerRepository}`, + "--set-string",`inferenceRouter.image.tag=${routerTag}`, + ...(dryRun?["--dry-run=server"]:[wait,"--timeout","8m"])],{stdio:"pipe"}); + return; + } + if(controller.metadata.annotations?.["meta.helm.sh/release-name"]) { + throw new Error("Controller reports Helm ownership that was not found; no template-mode adoption is allowed"); + } + const allowedRoles=["kars-sre-registrar","kars-sre-router-renew","kars-sre-private-diagnostics","kars-sre-retired-agent","kars-sre-authority-controller"]; + const objects=documents + .filter(obj=>(obj.kind==="CustomResourceDefinition"&&obj.metadata.name==="karssreregistrations.kars.azure.com") + || (["ValidatingAdmissionPolicy","ValidatingAdmissionPolicyBinding"].includes(obj.kind!)&&obj.metadata.name?.startsWith("kars-sre-")) + || (obj.kind==="ClusterRole"&&allowedRoles.includes(obj.metadata.name!)) + || (obj.kind==="ClusterRoleBinding"&&obj.metadata.name==="kars-sre-authority-controller")); + if(!objects.some(obj=>obj.kind==="CustomResourceDefinition"))throw new Error("Authority CRD is absent from the staged chart"); + const writes:Array<{object:ApiObject;existing?:ApiObject}>=[]; + const unchangedCrds:string[]=[]; + for(const object of objects) { + const existing=await get(execute,object.kind!.toLowerCase(),object.metadata.name!); + if(existing) { + const desired=object.spec??object.rules??{roleRef:object.roleRef,subjects:object.subjects}; + const current=existing.spec??existing.rules??{roleRef:existing.roleRef,subjects:existing.subjects}; + if(canonical(desired)===canonical(current)) { + if(object.kind==="CustomResourceDefinition")unchangedCrds.push(object.metadata.name!); + continue; + } + if(existing.metadata.annotations?.["kars.azure.com/sre-authority-staged"]!==namespace + || existing.metadata.annotations?.["kars.azure.com/sre-authority-release"]!==release) { + throw new Error(`Unowned authority object ${object.kind}/${object.metadata.name} differs; no objects were adopted`); + } + } + writes.push({object,existing}); + } + const containers=structuredClone(controller.spec?.template?.spec?.containers); + if(!Array.isArray(containers))throw new Error("Controller container specification is missing"); + const main=containers.find(container=>container.name==="controller"); + if(!main)throw new Error("Controller container identity is unrecognized"); + main.image=controllerImage; + main.env=(main.env??[]).filter((entry:{name:string})=>entry.name!=="INFERENCE_ROUTER_IMAGE"); + main.env.push({name:"INFERENCE_ROUTER_IMAGE",value:routerImage}); + if(dryRun) { + console.log(`Would verify/CAS-repair the action API prerequisite, stage ${writes.length} authority objects and CAS-update controller ${controller.metadata.uid}@${controller.metadata.resourceVersion}`); + return; + } + await stageAction(); + for(const name of unchangedCrds) { + await execute("kubectl",["wait","--for=condition=Established",`crd/${name}`,"--timeout=60s"],{stdio:"pipe"}); + } + for(const {object,existing} of writes.sort((a,b)=>Number(b.object.kind==="CustomResourceDefinition")-Number(a.object.kind==="CustomResourceDefinition"))) { + const annotations={...object.metadata.annotations, + "kars.azure.com/sre-authority-staged":namespace,"kars.azure.com/sre-authority-release":release}; + if(existing) { + await execute("kubectl",["patch",object.kind!.toLowerCase(),object.metadata.name!,"--type=merge","-p",JSON.stringify({ + ...object,metadata:{uid:existing.metadata.uid,resourceVersion:existing.metadata.resourceVersion,annotations}, + })],{stdio:"pipe"}); + } else { + await execute("kubectl",["create","-f","-"],{stdio:"pipe",input:JSON.stringify({ + ...object,metadata:{...object.metadata,annotations}, + })}); + } + if(object.kind==="CustomResourceDefinition") { + await execute("kubectl",["wait","--for=condition=Established",`crd/${object.metadata.name}`,"--timeout=60s"],{stdio:"pipe"}); + } + } + await execute("kubectl",["patch","deployment","kars-controller","-n",namespace,"--type=merge","-p",JSON.stringify({ + metadata:{uid:controller.metadata.uid,resourceVersion:controller.metadata.resourceVersion}, + spec:{template:{spec:{containers}}}, + })],{stdio:"pipe"}); + await execute("kubectl",["rollout","status","deployment/kars-controller","-n",namespace,"--timeout=8m"],{stdio:"pipe"}); +} diff --git a/cli/src/testing/shared-security-guards.test.ts b/cli/src/testing/shared-security-guards.test.ts new file mode 100644 index 000000000..6b02ce7f1 --- /dev/null +++ b/cli/src/testing/shared-security-guards.test.ts @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const root = new URL("../../../", import.meta.url); +const source = (path: string) => readFileSync(new URL(path, root), "utf8"); + +describe("shared Rust security guard coverage", () => { + it.each(["no-stubs.sh", "no-custom-crypto.sh"])("includes shared code in %s production paths", name => { + const paths = source(`ci/${name}`).match(/PROD_PATHS=\(([\s\S]*?)\n\)/)?.[1]; + expect(paths).toBeDefined(); + expect(paths).toContain("'shared/'"); + }); + + it("requires a capability audit for shared Rust changes without gating Markdown", () => { + const pattern = source("ci/security-audit-required.sh").match(/^CAP_RE='([^']+)'$/m)?.[1]; + expect(pattern).toBeDefined(); + const result = spawnSync("grep", ["-E", pattern!], { + encoding: "utf8", + input: [ + "shared/sre_privacy.rs", "shared/future/authority.rs", + "shared/README.md", "docs/shared/sre_privacy.rs", + ].join("\n"), + }); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "shared/sre_privacy.rs", "shared/future/authority.rs", + ]); + }); +}); diff --git a/cli/src/testing/sre-authority.test.ts b/cli/src/testing/sre-authority.test.ts new file mode 100644 index 000000000..bd439623d --- /dev/null +++ b/cli/src/testing/sre-authority.test.ts @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe,expect,it } from "vitest"; +import { parseAllDocuments } from "yaml"; + +const root=fileURLToPath(new URL("../../../",import.meta.url)); +const chart=fileURLToPath(new URL("../../../deploy/helm/kars",import.meta.url)); + +describe("SRE authority chart and mutation integration",()=>{ + it("escapes custom namespace fields in CEL without renaming the wire fields or weakening equality",()=>{ + const output=execFileSync("helm",["template","kars",chart,"--show-only","templates/crd-karssreregistration.yaml"],{encoding:"utf8"}); + const registration=parseAllDocuments(output).map(doc=>doc.toJSON()).find(Boolean); + const schema=registration.spec.versions[0].schema.openAPIV3Schema; + const spec=schema.properties.spec; + expect(spec.properties.sandbox.properties.namespace.type).toBe("string"); + expect(spec.properties.controller.properties.namespace.properties.name.type).toBe("string"); + expect(spec["x-kubernetes-validations"]).toEqual([{ + rule:"self.sandbox.__namespace__ == self.controller.__namespace__.name", + message:"SRE must be registered in its controller/release namespace", + }]); + expect(schema["x-kubernetes-validations"][0].rule).toBe("self.metadata.name == 'canonical'"); + }); + + it("creates a cluster registration and no default registrar or runtime privilege bindings",()=>{ + for(const enabled of [false,true]){ + const output=execFileSync("helm",["template","kars",chart,"--namespace","kars-system","--set",`sre.enabled=${enabled}`],{encoding:"utf8"}); + const docs=parseAllDocuments(output).map(doc=>doc.toJSON()).filter(Boolean); + const registration=docs.find(doc=>doc.kind==="CustomResourceDefinition"&&doc.metadata.name==="karssreregistrations.kars.azure.com"); + expect(registration.spec.scope).toBe("Cluster"); + const bindings=docs.filter(doc=>["ClusterRoleBinding","RoleBinding"].includes(doc.kind)); + expect(bindings.some(binding=>binding.roleRef.name==="kars-sre-registrar")).toBe(false); + expect(bindings.some(binding=>(binding.subjects??[]).some((subject:any)=> + subject.namespace==="kars-sre"&&["sandbox","sre-api-router"].includes(subject.name)))).toBe(false); + const controller=docs.find(doc=>doc.kind==="ClusterRole"&&doc.metadata.name==="kars-sre-authority-controller"); + expect(controller.rules.filter((rule:any)=>rule.resources.includes("karssreregistrations")) + .every((rule:any)=>rule.verbs.every((verb:string)=>["get","list","watch","use"].includes(verb)))).toBe(true); + const privileged=controller.rules.filter((rule:any)=> + rule.verbs.some((verb:string)=>["bind","escalate","*"].includes(verb))); + expect(privileged).toEqual([{ + apiGroups:["rbac.authorization.k8s.io"],resources:["clusterroles"], + resourceNames:["kars-sre-private-diagnostics","kars-sre-action-author","kars-sre-router-renew","kars-sre-reader"], + verbs:["bind"], + }]); + } + }); + + it("uses authorizer permissions rather than usernames for protected source/identity admission",()=>{ + const output=execFileSync("helm",["template","kars",chart,"--namespace","custom-controller"],{encoding:"utf8"}); + const policies=parseAllDocuments(output).map(doc=>doc.toJSON()).filter(doc=>doc?.kind==="ValidatingAdmissionPolicy"&&doc.metadata.name.startsWith("kars-sre-")); + expect(policies.length).toBeGreaterThan(5); + for(const name of ["kars-sre-source-authority","kars-sre-private-identity","kars-sre-registration-authority"]){ + const policy=policies.find(doc=>doc.metadata.name===name); + expect(policy.spec.failurePolicy).toBe("Fail"); + const expressions=JSON.stringify(policy.spec); + expect(expressions).toContain("authorizer.group('kars.azure.com')"); + expect(expressions).not.toContain("request.userInfo.username"); + } + }); + + it("places normal mutation and rollback checks before their owning writes",()=>{ + const checks=[ + ["cli/src/commands/push-apply.ts","await assertSafeMutation(execute)","const artifacts ="], + ["cli/src/commands/up/fast_upgrade.ts","await assertSafeMutation(execa)","const helmArgs ="], + ["cli/src/commands/up.ts","await assertSafeMutation(execa)","const helmArgs ="], + ["cli/src/commands/dev/local-k8s.ts","await assertSafeMutation(","const credsOverlay = await provisionDevCreds"], + ["cli/src/commands/upgrade.ts","await assertRollbackSafe(execa)","await execa(\"helm\", [\"rollback\""], + ]; + for(const [path,gate,write] of checks){ + const source=readFileSync(`${root}/${path}`,"utf8"); + expect(source.indexOf(gate),path).toBeGreaterThanOrEqual(0); + expect(source.indexOf(gate),path).toBeLessThan(source.indexOf(write)); + } + }); + + it("unconditionally denies legacy token Secret creation and both old/new update transitions under arbitrary names",()=>{ + const output=execFileSync("helm",["template","kars",chart],{encoding:"utf8"}); + const docs=parseAllDocuments(output).map(doc=>doc.toJSON()).filter(Boolean); + const policy=docs.find(doc=>doc.kind==="ValidatingAdmissionPolicy"&&doc.metadata.name==="kars-sre-no-legacy-tokens"); + expect(policy.spec.matchConstraints.resourceRules).toEqual([{ + apiGroups:[""],apiVersions:["v1"],operations:["CREATE","UPDATE"],resources:["secrets"], + }]); + const expression=policy.spec.validations[0].expression; + expect(expression).toContain("oldObject"); + expect(expression).toContain("object.?type"); + expect(expression).toContain("kubernetes.io/service-account.name"); + expect(expression).toContain("kubernetes.io/service-account-token"); + expect(expression).not.toContain("metadata.name"); + expect(expression).not.toContain("authorizer"); + expect(docs.find(doc=>doc.kind==="ValidatingAdmissionPolicyBinding" + &&doc.metadata.name===policy.metadata.name).spec.validationActions).toContain("Deny"); + }); +}); diff --git a/controller/Cargo.toml b/controller/Cargo.toml index 3a359e94f..1216bee75 100644 --- a/controller/Cargo.toml +++ b/controller/Cargo.toml @@ -68,6 +68,8 @@ oci-client = { version = "=0.16.1", default-features = false, features = ["rustl # refuse to auto-detect and panic on first TLS handshake. Pin to # `aws-lc-rs` to align with the rest of the workspace. rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } +rcgen.workspace = true +time.workspace = true regex = "1.12.3" [dev-dependencies] diff --git a/controller/src/kars_sre_action.rs b/controller/src/kars_sre_action.rs index 344649adc..3fdee9bf8 100644 --- a/controller/src/kars_sre_action.rs +++ b/controller/src/kars_sre_action.rs @@ -140,9 +140,19 @@ pub struct ActionSpec { /// - ScaleDeployment: {namespace, name, replicas} /// - RolloutRestart: {namespace, kind, name} /// - DeletePod: {namespace, name} + #[schemars(schema_with = "action_params_schema")] pub params: std::collections::BTreeMap, } +fn action_params_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + // Boolean additionalProperties exposes a nil-schema adapter in Kubernetes + // 1.31's VAP type checker. Preserve arbitrary JSON without that adapter. + schemars::json_schema!({ + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }) +} + /// Operator decision payload. #[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq)] #[serde(rename_all = "camelCase")] @@ -192,3 +202,34 @@ pub struct KarsSREActionStatus { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub conditions: Vec, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sre_action_params_schema_preserves_json_without_boolean_additional_properties() { + let crd = serde_json::to_value(crate::crd_validations::kars_sre_action_crd()).unwrap(); + let spec = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let params = &spec["properties"]["action"]["properties"]["params"]; + assert_eq!(params["type"], "object"); + assert_eq!(params["x-kubernetes-preserve-unknown-fields"], true); + assert!(params.get("additionalProperties").is_none()); + assert!(spec.get("x-kubernetes-preserve-unknown-fields").is_none()); + assert_eq!( + spec["properties"]["approval"]["properties"]["state"]["type"], + "string" + ); + } + + #[test] + fn sre_action_params_schema_does_not_change_json_wire_values() { + let wire = serde_json::json!({ + "type": "ScaleDeployment", + "params": {"namespace": "example", "name": "demo", "replicas": 1, + "nested": {"array": [true, null, 1, "text"], "object": {"key": "value"}}} + }); + let action: ActionSpec = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(action).unwrap(), wire); + } +} diff --git a/controller/src/main.rs b/controller/src/main.rs index 6a8a9bb4f..01ff8dbdb 100644 --- a/controller/src/main.rs +++ b/controller/src/main.rs @@ -73,6 +73,10 @@ mod policy_fetcher; mod providers; mod reconciler; mod signer_policy; +mod sre_authority; +#[path = "../../shared/sre_privacy.rs"] +mod sre_privacy; +mod sre_registration; mod status; mod task_models; mod team_commons; @@ -299,6 +303,10 @@ async fn main() -> Result<()> { let client = client.clone(); tokio::spawn(async move { kars_sre_action_reconciler::run(client).await }) }; + let sre_authority_handle = { + let client = client.clone(); + tokio::spawn(async move { sre_authority::run(client).await }) + }; let auth_config_handle = { // KarsAuthConfig reconciler — materialises the sidecar env // ConfigMap when an operator installs the tenant trust anchor @@ -458,6 +466,9 @@ async fn main() -> Result<()> { res = kars_sre_action_handle => { res??; } + res = sre_authority_handle => { + res?; + } res = auth_config_handle => { // auth-config reconciler exiting is non-fatal (it sleeps // forever when the CRD is absent), but we propagate any diff --git a/controller/src/providers/mod.rs b/controller/src/providers/mod.rs index e64ac9a9d..fcb4259a0 100644 --- a/controller/src/providers/mod.rs +++ b/controller/src/providers/mod.rs @@ -29,6 +29,7 @@ /// Governance Receipt signing (kars Bridge Inc 3). Allowlisted crypto /// wrapper: Ed25519 over DSSE. See the module docs for the V0 trust model. pub mod signing; +pub mod sre_tls; #[allow(unused_imports)] pub mod field_managers { diff --git a/controller/src/providers/signing.rs b/controller/src/providers/signing.rs index bcc0e5f05..c1e481932 100644 --- a/controller/src/providers/signing.rs +++ b/controller/src/providers/signing.rs @@ -177,6 +177,13 @@ pub fn content_digest(bytes: &[u8]) -> String { format!("sha256:{}", &sha256_hex(bytes)[..32]) } +/// Opaque service credentials; callers must keep them out of agent authority +/// except where the credential deliberately grants only a filtered interface. +pub fn generate_service_token() -> String { + use rand::distr::{Alphanumeric, SampleString}; + Alphanumeric.sample_string(&mut rand::rng(), 64) +} + /// DSSE Pre-Authentication Encoding: /// `"DSSEv1" SP len(type) SP type SP len(body) SP body`. /// diff --git a/controller/src/providers/sre_tls.rs b/controller/src/providers/sre_tls.rs new file mode 100644 index 000000000..bfd1e5d80 --- /dev/null +++ b/controller/src/providers/sre_tls.rs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Standard X.509 issuance for a pod-local compatibility endpoint. + +use rcgen::{ + BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose, +}; + +pub struct Identity { + pub ca: String, + pub certificate: String, + pub private_key: String, + pub expires_at: i64, +} + +pub fn issue() -> Result { + let now = time::OffsetDateTime::now_utc(); + let not_before = now - time::Duration::hours(1); + let expiry = now + time::Duration::days(30); + let mut root = CertificateParams::new(Vec::::new()) + .map_err(|_| "SRE CA parameters are invalid")?; + root.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + root.distinguished_name + .push(rcgen::DnType::CommonName, "Kars SRE loopback CA"); + root.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + root.not_before = not_before; + root.not_after = expiry; + let root_key = KeyPair::generate().map_err(|_| "SRE CA key generation failed")?; + let ca = root + .self_signed(&root_key) + .map_err(|_| "SRE CA issuance failed")?; + let mut leaf = CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]) + .map_err(|_| "SRE TLS parameters are invalid")?; + leaf.not_before = not_before; + leaf.distinguished_name + .push(rcgen::DnType::CommonName, "Kars SRE loopback API"); + leaf.not_after = expiry; + leaf.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + leaf.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + let key = KeyPair::generate().map_err(|_| "SRE TLS key generation failed")?; + let certificate = leaf + .signed_by(&key, &ca, &root_key) + .map_err(|_| "SRE TLS issuance failed")?; + Ok(Identity { + ca: ca.pem(), + certificate: certificate.pem(), + private_key: key.serialize_pem(), + expires_at: expiry.unix_timestamp(), + }) +} diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 8e86917fe..3cf68b9ca 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -47,6 +47,7 @@ pub(crate) mod trustgraph_mount; use mcp_egress::mcp_egress_rule; mod pod_spec; +mod sre_egress; pub(crate) use pod_spec::{ build_egress_guard_command, build_pod_security_context, isolation_scheduling, sandbox_node_selector_from, @@ -199,22 +200,8 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result projection, + Err(detail) => { + crate::status::stamp_degraded(client, &sandbox, &name, "SREAuthorityNotReady", &detail) + .await; + return Ok(Action::requeue(Duration::from_secs(20))); + } + }; let credentials = credential_sources::reconcile( client, &sandbox, @@ -942,25 +938,14 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result().unwrap_or(443)}] - })); + // Policy enforcement may observe the API's post-DNAT endpoint IP/port. + // Only registered routers receive exact targets; UID 1000 stays locked down. + if sre_projection.is_some() { + egress_rules.extend( + sre_egress::rules(client, &apiserver_ip, &apiserver_port) + .await + .map_err(ReconcileError::Configuration)?, + ); } // Add user-defined allowed endpoints (for the inference-router to reach @@ -1916,9 +1901,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, + >(annotations)?); + } if let Some(namespace) = owned_namespace.as_ref() { credentials.decorate(&mut deployment, &sandbox, namespace); } @@ -3122,7 +3106,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result Result<()> { crate::watch_config::bounded(), deployment_to_sandbox_ref, ) + .watches( + Api::::all(ctx.client.clone()), + crate::watch_config::bounded(), + |registration| { + Some( + kube::runtime::reflector::ObjectRef::new(®istration.spec.sandbox.name) + .within(®istration.spec.sandbox.namespace), + ) + }, + ) .run( |x, ctx| async move { crate::metrics::observe_reconcile("KarsSandbox", reconcile(x, ctx)).await diff --git a/controller/src/reconciler/pod_spec.rs b/controller/src/reconciler/pod_spec.rs index 0e4c70e3a..22925e184 100644 --- a/controller/src/reconciler/pod_spec.rs +++ b/controller/src/reconciler/pod_spec.rs @@ -94,28 +94,14 @@ pub(crate) fn sandbox_node_selector_from( /// Build the egress-guard init-container command. /// -/// Standard sandboxes (every kind except SRE) get the full lockdown: +/// Every sandbox, including SRE, gets the full lockdown: /// UID 1000 → loopback + DNS allowed, everything else dropped, with /// :80/:443 NAT-redirected to the inference-router on :8444 for L7 /// policy + audit. /// -/// SRE-mode sandboxes (labelled `kars.azure.com/role=sre`) get ONE -/// extra rule inserted into the OUTPUT NAT chain BEFORE the generic -/// REDIRECT: apiserver-bound traffic (KUBERNETES_SERVICE_HOST : -/// KUBERNETES_SERVICE_PORT_HTTPS, both kubelet-auto-injected envs) -/// is RETURNed — i.e. NOT NAT'd to :8444 — so the SRE plugin's K8s -/// API client (sre_kube.py) can hit the apiserver directly with its -/// projected SA token. -/// -/// The K8s audit log is the audit surface for these apiserver calls -/// (the router's L7 audit doesn't capture them, but K8s audit is -/// stronger — every call carries the SA identity and the verb). -/// -/// Privilege-containment design: this capability is uniquely held by -/// the SRE sandbox per the proposal §7.8. Future Slice 3 will add -/// ValidatingAdmissionPolicies to gate WHO can apply the -/// `role=sre` label (only chart-installer SAs; see §7.8.10 design). -pub(crate) fn build_egress_guard_command(is_sre_sandbox: bool) -> String { +/// SRE diagnostics use a registered, filtered loopback HTTPS service instead +/// of direct Kubernetes access with an agent-held privileged credential. +pub(crate) fn build_egress_guard_command(_is_sre_sandbox: bool) -> String { let mut cmd = String::with_capacity(1024); // Filter chain (OUTPUT): UID 1000 → allow loopback + DNS + // established, then DROP. Same for every sandbox kind. @@ -126,36 +112,8 @@ pub(crate) fn build_egress_guard_command(is_sre_sandbox: bool) -> String { "iptables -A OUTPUT -m owner --uid-owner 1000 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT && " ); - // SRE-mode-only: filter-chain ACCEPT for apiserver-bound traffic. - // The filter chain runs AFTER the NAT chain — the NAT-bypass RETURN - // below just decides "don't redirect", but the filter chain's DROP - // (next rule) would still kill the packet. We have to ACCEPT it - // here BEFORE the catch-all DROP. - if is_sre_sandbox { - cmd.push_str( - "iptables -A OUTPUT -m owner --uid-owner 1000 \ - -d \"${KUBERNETES_SERVICE_HOST}\" \ - -p tcp --dport \"${KUBERNETES_SERVICE_PORT_HTTPS:-443}\" \ - -j ACCEPT && ", - ); - } - cmd.push_str("iptables -A OUTPUT -m owner --uid-owner 1000 -j DROP && "); - // SRE-mode-only: NAT-chain apiserver bypass. Inserted BEFORE the - // generic :443 REDIRECT so apiserver traffic short-circuits to the - // real upstream rather than the router. KUBERNETES_SERVICE_HOST - // and KUBERNETES_SERVICE_PORT_HTTPS are auto-injected by the - // kubelet on every container (including init containers). - if is_sre_sandbox { - cmd.push_str( - "iptables -t nat -A OUTPUT -m owner --uid-owner 1000 \ - -d \"${KUBERNETES_SERVICE_HOST}\" \ - -p tcp --dport \"${KUBERNETES_SERVICE_PORT_HTTPS:-443}\" \ - -j RETURN && ", - ); - } - // NAT chain (OUTPUT): :80/:443 → REDIRECT to :8444 (transparent // proxy in the inference-router sidecar). Same for every sandbox. cmd.push_str( @@ -165,15 +123,7 @@ pub(crate) fn build_egress_guard_command(is_sre_sandbox: bool) -> String { "iptables -t nat -A OUTPUT -m owner --uid-owner 1000 ! -o lo -p tcp --dport 443 -j REDIRECT --to-port 8444 && " ); - if is_sre_sandbox { - cmd.push_str( - "echo 'egress-guard: UID 1000 → transparent proxy on :8444 + apiserver bypass (SRE mode)'" - ); - } else { - cmd.push_str( - "echo 'egress-guard: UID 1000 → transparent proxy on :8444 (learn + enforce)'", - ); - } + cmd.push_str("echo 'egress-guard: UID 1000 → transparent proxy on :8444 (learn + enforce)'"); cmd } @@ -193,53 +143,11 @@ mod egress_guard_tests { } #[test] - fn sre_sandbox_inserts_apiserver_bypass_before_redirect() { + fn sre_uses_the_same_network_lockdown_without_an_apiserver_bypass() { let cmd = build_egress_guard_command(true); - // The bypass MUST come before the :443 REDIRECT — otherwise - // the REDIRECT wins (iptables -A appends; rules evaluate in - // order) and the bypass is dead code. - let bypass_pos = cmd - .find("-t nat -A OUTPUT -m owner --uid-owner 1000 -d \"${KUBERNETES_SERVICE_HOST}\"") - .or_else(|| cmd.find("-t nat -A OUTPUT -m owner --uid-owner 1000 \t\t\t -d \"${KUBERNETES_SERVICE_HOST}\"")) - .or_else(|| { - // Match the NAT-chain bypass specifically (not the filter ACCEPT) - cmd.match_indices("-t nat -A OUTPUT") - .find(|(i, _)| cmd[*i..].contains("KUBERNETES_SERVICE_HOST")) - .map(|(i, _)| i) - }) - .expect("NAT-chain bypass rule missing"); - let redirect_pos = cmd - .find("--dport 443 -j REDIRECT") - .expect("redirect rule missing"); - assert!( - bypass_pos < redirect_pos, - "NAT bypass at {bypass_pos} must precede redirect at {redirect_pos}" - ); - assert!(cmd.contains("apiserver bypass (SRE mode)")); - - // ALSO check the filter-chain ACCEPT exists BEFORE the DROP — this - // was the bug we hit live: NAT bypass alone wasn't enough because - // the filter chain's DROP for UID 1000 killed the packet anyway. - let filter_accept = cmd - .find( - "-A OUTPUT -m owner --uid-owner 1000 -d \"${KUBERNETES_SERVICE_HOST}\"", - ) - .or_else(|| { - cmd.match_indices("-A OUTPUT -m owner --uid-owner 1000") - .find(|(i, _)| { - let tail = &cmd[*i..*i + 200.min(cmd.len() - *i)]; - tail.contains("KUBERNETES_SERVICE_HOST") && tail.contains("-j ACCEPT") - }) - .map(|(i, _)| i) - }) - .expect("filter-chain ACCEPT for apiserver missing"); - let filter_drop = cmd - .find("-A OUTPUT -m owner --uid-owner 1000 -j DROP") - .expect("filter DROP rule missing"); - assert!( - filter_accept < filter_drop, - "filter ACCEPT at {filter_accept} must precede DROP at {filter_drop}" - ); + assert_eq!(cmd, build_egress_guard_command(false)); + assert!(!cmd.contains("KUBERNETES_SERVICE_HOST")); + assert!(!cmd.contains("-j RETURN")); } #[test] diff --git a/controller/src/reconciler/sre_egress.rs b/controller/src/reconciler/sre_egress.rs new file mode 100644 index 000000000..7feb2e083 --- /dev/null +++ b/controller/src/reconciler/sre_egress.rs @@ -0,0 +1,270 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use k8s_openapi::{ + api::core::v1::{Endpoints, Service}, + apimachinery::pkg::apis::meta::v1::ObjectMeta, +}; +use kube::{Api, Client}; +use serde_json::{Value, json}; +use std::{collections::BTreeSet, net::IpAddr}; + +const MAX_ENDPOINTS: usize = 32; + +fn read_error(resource: &'static str, error: kube::Error) -> String { + match error { + kube::Error::Api(response) => { + format!( + "Read canonical API {resource}: Kubernetes status {}", + response.code + ) + } + _ => format!("Read canonical API {resource}: Kubernetes transport failure"), + } +} + +pub(super) async fn rules( + client: &Client, + service_host: &str, + service_port: &str, +) -> Result, String> { + let service = Api::::namespaced(client.clone(), "default") + .get("kubernetes") + .await + .map_err(|error| read_error("Service", error))?; + let endpoints = Api::::namespaced(client.clone(), "default") + .get("kubernetes") + .await + .map_err(|error| read_error("Endpoints", error))?; + from_objects(&service, &endpoints, service_host, service_port) +} + +fn canonical(metadata: &ObjectMeta) -> bool { + metadata.name.as_deref() == Some("kubernetes") + && metadata.namespace.as_deref() == Some("default") + && metadata + .uid + .as_deref() + .is_some_and(|value| !value.is_empty()) + && metadata + .resource_version + .as_deref() + .is_some_and(|value| !value.is_empty()) + && metadata.deletion_timestamp.is_none() +} + +fn address(value: &str) -> Result { + let ip: IpAddr = value + .parse() + .map_err(|_| "Invalid API endpoint IP address")?; + if ip.is_unspecified() + || ip.is_loopback() + || ip.is_multicast() + || matches!(ip, IpAddr::V4(ip) if ip.is_broadcast() || ip.is_link_local()) + || matches!(ip, IpAddr::V6(ip) if ip.is_unicast_link_local()) + { + return Err("API endpoint must be a routable unicast IP".into()); + } + Ok(ip) +} + +fn port(value: i32) -> Result { + u16::try_from(value) + .ok() + .filter(|value| *value != 0) + .ok_or_else(|| "Invalid API HTTPS port".into()) +} + +fn rule(ip: IpAddr, port: u16) -> Value { + let prefix = if ip.is_ipv4() { 32 } else { 128 }; + json!({ + "to": [{"ipBlock": {"cidr": format!("{ip}/{prefix}")}}], + "ports": [{"protocol": "TCP", "port": port}], + }) +} + +fn from_objects( + service: &Service, + endpoints: &Endpoints, + service_host: &str, + service_port: &str, +) -> Result, String> { + if !canonical(&service.metadata) || !canonical(&endpoints.metadata) { + return Err("Canonical API Service/Endpoints identity is missing or terminating".into()); + } + let host = address(service_host)?; + let configured_port = port( + service_port + .parse() + .map_err(|_| "Invalid configured API HTTPS port")?, + )?; + let spec = service + .spec + .as_ref() + .ok_or("Canonical API Service spec is missing")?; + if spec.cluster_ip.as_deref().map(address).transpose()? != Some(host) + || !spec + .ports + .as_deref() + .unwrap_or_default() + .iter() + .any(|entry| { + entry.name.as_deref() == Some("https") + && entry.protocol.as_deref().unwrap_or("TCP") == "TCP" + && entry.port == i32::from(configured_port) + }) + { + return Err("Canonical API Service does not match the controller's HTTPS target".into()); + } + let mut targets = BTreeSet::new(); + for subset in endpoints.subsets.as_deref().unwrap_or_default() { + for entry in subset.ports.as_deref().unwrap_or_default() { + if entry.name.as_deref() != Some("https") { + continue; + } + if entry.protocol.as_deref().unwrap_or("TCP") != "TCP" { + return Err("API HTTPS endpoint must use TCP".into()); + } + let endpoint_port = port(entry.port)?; + // Only ready addresses are eligible; never include notReadyAddresses. + for entry in subset.addresses.as_deref().unwrap_or_default() { + let ip = address(&entry.ip)?; + if ip.is_ipv4() != host.is_ipv4() { + continue; + } + targets.insert((ip, endpoint_port)); + if targets.len() > MAX_ENDPOINTS { + return Err("Canonical API endpoint inventory exceeds its bounded limit".into()); + } + } + } + } + if targets.is_empty() { + return Err("Canonical API Service has no ready HTTPS endpoints".into()); + } + let mut rules = vec![rule(host, configured_port)]; + rules.extend( + targets + .into_iter() + .filter(|target| *target != (host, configured_port)) + .map(|(ip, port)| rule(ip, port)), + ); + Ok(rules) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixtures() -> (Service, Endpoints) { + let metadata = + json!({"name":"kubernetes","namespace":"default","uid":"live","resourceVersion":"1"}); + ( + serde_json::from_value(json!({"metadata":metadata,"spec":{ + "clusterIP":"10.96.0.1","ports":[{"name":"https","port":443,"protocol":"TCP"}]}})) + .unwrap(), + serde_json::from_value(json!({"metadata":metadata,"subsets":[{ + "addresses":[{"ip":"172.18.0.2"},{"ip":"172.18.0.2"}], + "notReadyAddresses":[{"ip":"172.18.0.99"}], + "ports":[{"name":"https","port":6443,"protocol":"TCP"}]}]})) + .unwrap(), + ) + } + + #[test] + fn exact_service_and_post_dnat_targets_are_paired_deduplicated_and_ready_only() { + let (service, endpoints) = fixtures(); + assert_eq!( + from_objects(&service, &endpoints, "10.96.0.1", "443").unwrap(), + vec![ + rule("10.96.0.1".parse().unwrap(), 443), + rule("172.18.0.2".parse().unwrap(), 6443) + ] + ); + } + + #[test] + fn ipv6_uses_only_exact_ready_targets_of_the_service_family() { + let (mut service, mut endpoints) = fixtures(); + service.spec.as_mut().unwrap().cluster_ip = Some("fd00::1".into()); + endpoints.subsets.as_mut().unwrap()[0] + .addresses + .as_mut() + .unwrap()[0] + .ip = "fd01::2".into(); + assert_eq!( + from_objects(&service, &endpoints, "fd00::1", "443").unwrap(), + vec![ + rule("fd00::1".parse().unwrap(), 443), + rule("fd01::2".parse().unwrap(), 6443) + ] + ); + } + + #[test] + fn missing_identity_mismatched_service_and_unready_inventory_fail_closed() { + let (service, endpoints) = fixtures(); + assert!(from_objects(&service, &endpoints, "10.96.0.2", "443").is_err()); + assert!(from_objects(&service, &endpoints, "10.96.0.1", "444").is_err()); + assert!(from_objects(&service, &endpoints, "10.96.0.1", "invalid").is_err()); + let mut invalid = service.clone(); + invalid.metadata.namespace = Some("other".into()); + assert!(from_objects(&invalid, &endpoints, "10.96.0.1", "443").is_err()); + let mut invalid = endpoints.clone(); + invalid.metadata.uid = None; + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + let mut invalid = endpoints.clone(); + invalid.subsets.as_mut().unwrap()[0].addresses = None; + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + + #[test] + fn malformed_addresses_ports_or_protocols_never_produce_partial_rules() { + let (service, endpoints) = fixtures(); + for ip in [ + "invalid", + "0.0.0.0", + "127.0.0.1", + "169.254.169.254", + "224.0.0.1", + "255.255.255.255", + ] { + let mut invalid = endpoints.clone(); + invalid.subsets.as_mut().unwrap()[0] + .addresses + .as_mut() + .unwrap()[0] + .ip = ip.into(); + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + for value in [0, -1, 65536] { + let mut invalid = endpoints.clone(); + invalid.subsets.as_mut().unwrap()[0].ports.as_mut().unwrap()[0].port = value; + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + let mut invalid = endpoints; + invalid.subsets.as_mut().unwrap()[0].ports.as_mut().unwrap()[0].protocol = + Some("UDP".into()); + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + + #[test] + fn endpoint_expansion_is_bounded_and_read_errors_never_echo_api_bodies() { + let (service, mut endpoints) = fixtures(); + endpoints.subsets.as_mut().unwrap()[0].addresses = Some( + (1..=33) + .map(|index| { + serde_json::from_value(json!({"ip":format!("172.18.1.{index}")})).unwrap() + }) + .collect(), + ); + assert!(from_objects(&service, &endpoints, "10.96.0.1", "443").is_err()); + let error = kube::Error::Api(serde_json::from_value(json!({ + "status":"Failure","message":"private-response-must-not-log","reason":"Forbidden","code":403 + })).unwrap()); + assert_eq!( + read_error("Endpoints", error), + "Read canonical API Endpoints: Kubernetes status 403" + ); + } +} diff --git a/controller/src/sre_authority.rs b/controller/src/sre_authority.rs new file mode 100644 index 000000000..43bc5e191 --- /dev/null +++ b/controller/src/sre_authority.rs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Registrar-authorized migration and private SRE Kubernetes identity. + +mod admission; +mod bindings; +mod credential_guard; +mod credentials; +mod live; +mod migration; +pub(crate) mod pod; +#[cfg(test)] +mod privacy_tests; +#[cfg(test)] +mod retirement_tests; +#[cfg(test)] +mod tests; + +use crate::sre_registration::{KarsSRERegistration, NAME, RegistrationStatus}; +use crate::status::conditions; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; +use kube::{ + Api, Client, ResourceExt, + api::{Patch, PatchParams}, +}; + +pub(crate) use live::{api_error, check_secret_denial, privacy_epoch}; + +fn registration_conditions( + prior: &[Condition], + generation: Option, + phase: &str, + detail: Option<&str>, +) -> Vec { + let (reason, message) = match phase { + "Ready" => ("AuthorityReady", "Reviewed SRE authority is ready"), + "Retired" => ("AuthorityRetired", "Private SRE authority has been retired"), + "Provisioning" => ("Provisioning", "Private SRE identity is being provisioned"), + "Migrating" => ( + "Migrating", + "Reviewed SRE grants and consumers are migrating", + ), + _ => ("AuthorityBlocked", "SRE authority cannot be established"), + }; + let mut result = prior.to_vec(); + for (kind, active) in [ + (conditions::TYPE_READY, phase == "Ready"), + ( + conditions::TYPE_PROGRESSING, + matches!(phase, "Migrating" | "Provisioning"), + ), + ( + conditions::TYPE_DEGRADED, + !matches!(phase, "Ready" | "Retired" | "Migrating" | "Provisioning"), + ), + ] { + conditions::set( + &mut result, + conditions::preserve_transition_time( + conditions::find(prior, kind), + kind, + if active { + conditions::status::TRUE + } else { + conditions::status::FALSE + }, + reason, + detail.unwrap_or(message), + generation, + ), + ); + } + result +} + +async fn status( + client: &Client, + reg: &KarsSRERegistration, + phase: &str, + detail: Option, + service_account_uid: Option, +) -> Result<(), String> { + let api: Api = Api::all(client.clone()); + let status = RegistrationStatus { + phase: phase.into(), + observed_generation: reg.metadata.generation.unwrap_or_default(), + conditions: registration_conditions( + reg.status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or_default(), + reg.metadata.generation, + phase, + detail.as_deref(), + ), + privacy_epoch: (phase == "Ready").then(|| reg.epoch()), + router_service_account_uid: if phase == "Retired" { + None + } else { + service_account_uid.or_else(|| { + reg.status + .as_ref() + .and_then(|status| status.router_service_account_uid.clone()) + }) + }, + legacy_secret_access_denied: matches!(phase, "Ready" | "Retired"), + privacy_revision: matches!(phase, "Ready" | "Retired") + .then(|| crate::sre_privacy::REVISION.into()), + detail, + }; + let mut status_value = + serde_json::to_value(&status).map_err(|_| "SRE authority status serialization failed")?; + status_value["privacyEpoch"] = serde_json::json!(status.privacy_epoch); + status_value["privacyRevision"] = serde_json::json!(status.privacy_revision); + status_value["routerServiceAccountUid"] = serde_json::json!(status.router_service_account_uid); + api.patch_status( + NAME, + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "metadata":{"uid":reg.metadata.uid,"resourceVersion":reg.metadata.resource_version}, + "status":status_value, + })), + ) + .await + .map_err(|error| api_error("Publish SRE authority status", error))?; + Ok(()) +} + +pub async fn reconcile(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + reg.validate()?; + let result = reconcile_inner(client, reg).await; + if let Err(error) = &result { + let waiting = migration::is_waiting(error); + let revocation = if waiting { + Ok(()) + } else { + bindings::revoke_private(client, reg).await + }; + let detail = match revocation { + Ok(()) => error.clone(), + Err(revocation) => { + format!("{error}; owned private authority revocation failed: {revocation}") + } + }; + let detail = if !waiting + && reg + .status + .as_ref() + .is_some_and(|status| status.router_service_account_uid.is_some()) + { + match credentials::retire(client, reg).await { + Ok(()) => detail, + Err(error) => { + format!("{detail}; owned private credential retirement failed: {error}") + } + } + } else { + detail + }; + status( + client, + reg, + if waiting { "Migrating" } else { "Blocked" }, + Some(detail), + None, + ) + .await?; + } + result +} + +async fn reconcile_inner(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + reg.validate()?; + if !reg.spec.enabled + && reg.status.as_ref().is_some_and(|status| { + status.phase == "Retired" + && status.observed_generation == reg.metadata.generation.unwrap_or_default() + && status.privacy_revision.as_deref() == Some(crate::sre_privacy::REVISION) + }) + { + migration::stop_registered_consumer_for_retirement(client, reg).await?; + return check_secret_denial(client, ®.spec.runtime_namespace.name).await; + } + if !reg.spec.enabled { + migration::stop_registered_consumer_for_retirement(client, reg).await?; + bindings::revoke_private(client, reg).await?; + credentials::retire(client, reg).await?; + check_secret_denial(client, ®.spec.runtime_namespace.name).await?; + return status(client, reg, "Retired", None, None).await; + } + let authority = live::verify(client, reg).await?; + admission::verify(client).await?; + credential_guard::scan(client, reg).await?; + // Validate the full review set before retiring even the first grant. + let reviewed = bindings::review(client, reg).await?; + migration::validate_consumer(client, reg).await?; + bindings::validate_private_targets(client, reg).await?; + credentials::review_targets(client, reg).await?; + bindings::retire_legacy(client, reg, &reviewed).await?; + check_secret_denial(client, ®.spec.runtime_namespace.name).await?; + migration::stop_legacy_consumer(client, reg).await?; + let service_account = credentials::ensure_service_account(client, reg, &authority).await?; + let refreshed = if reg + .status + .as_ref() + .and_then(|status| status.router_service_account_uid.as_ref()) + != service_account.metadata.uid.as_ref() + { + status(client, reg, "Provisioning", None, service_account.uid()).await?; + Api::::all(client.clone()) + .get(NAME) + .await + .map_err(|error| api_error("Refresh provisioning registration", error))? + } else { + reg.clone() + }; + if refreshed.metadata.uid != reg.metadata.uid + || refreshed.metadata.generation != reg.metadata.generation + { + return Err("Registration changed while provisioning private identity".into()); + } + let reg = &refreshed; + bindings::grant_private(client, reg, &service_account).await?; + // No private material is minted until the old principal is actually denied. + check_secret_denial(client, ®.spec.runtime_namespace.name).await?; + credential_guard::scan(client, reg).await?; + live::verify(client, reg).await?; + credentials::ensure(client, reg, &authority, &service_account).await?; + migration::rotate_owned_control_credentials(client, reg).await?; + status(client, reg, "Ready", None, service_account.uid()).await +} + +pub async fn run(client: Client) { + let api: Api = Api::all(client.clone()); + loop { + match api.get_opt(NAME).await { + Ok(Some(reg)) => { + if let Err(error) = reconcile(&client, ®).await { + tracing::warn!(registration = NAME, error = %error, "SRE authority is not ready"); + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!(error = %api_error("Read SRE registration", error), "SRE authority unavailable") + } + } + tokio::time::sleep(std::time::Duration::from_secs(20)).await; + } +} + +#[cfg(test)] +mod status_tests { + use super::*; + + #[test] + fn registration_phases_publish_truthful_standard_conditions() { + for (phase, expected) in [ + ("Ready", ["True", "False", "False"]), + ("Retired", ["False", "False", "False"]), + ("Migrating", ["False", "True", "False"]), + ("Provisioning", ["False", "True", "False"]), + ("Blocked", ["False", "False", "True"]), + ("UnknownPhase", ["False", "False", "True"]), + ] { + let result = registration_conditions(&[], Some(3), phase, None); + assert_eq!(result.len(), 3); + for (kind, expected) in ["Ready", "Progressing", "Degraded"] + .into_iter() + .zip(expected) + { + let condition = conditions::find(&result, kind).unwrap(); + assert_eq!(condition.status, expected, "{phase}/{kind}"); + assert_eq!(condition.observed_generation, Some(3)); + assert!(!condition.reason.is_empty()); + assert!(!condition.message.is_empty()); + } + } + } + + #[test] + fn repeated_status_preserves_transition_time_and_unrelated_conditions() { + let mut prior = registration_conditions(&[], Some(1), "Migrating", None); + conditions::set( + &mut prior, + conditions::new_condition("CustomEvidence", "True", "Observed", "preserve", Some(1)), + ); + let next = registration_conditions(&prior, Some(2), "Migrating", Some("Still draining")); + assert_eq!(next.len(), 4); + assert_eq!( + conditions::find(&next, "CustomEvidence"), + conditions::find(&prior, "CustomEvidence") + ); + for kind in ["Ready", "Progressing", "Degraded"] { + let before = conditions::find(&prior, kind).unwrap(); + let after = conditions::find(&next, kind).unwrap(); + assert_eq!(before.last_transition_time, after.last_transition_time); + assert_eq!(after.observed_generation, Some(2)); + assert_eq!(after.message, "Still draining"); + } + } +} diff --git a/controller/src/sre_authority/admission.rs b/controller/src/sre_authority/admission.rs new file mode 100644 index 000000000..2723ebc33 --- /dev/null +++ b/controller/src/sre_authority/admission.rs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::api_error; +use k8s_openapi::api::admissionregistration::v1::{ + ValidatingAdmissionPolicy, ValidatingAdmissionPolicyBinding, +}; +use kube::{Api, Client}; + +pub(super) const POLICIES: &[&str] = &[ + "kars-sre-source-authority", + "kars-sre-registration-authority", + "kars-sre-private-identity", + "kars-sre-binding-authority", + "kars-sre-private-material", + "kars-sre-no-legacy-tokens", + "kars-sre-source-retirement", + "kars-sre-pending-proposals", + "kars-sre-consumer-authority", + "kars-sre-private-mounts", + "kars-sre-private-workloads", + "kars-sre-private-cronjobs", + "kars-sre-private-connect", + "kars-sre-role-authority", +]; + +pub(super) async fn verify(client: &Client) -> Result<(), String> { + let policies: Api = Api::all(client.clone()); + let bindings: Api = Api::all(client.clone()); + for name in POLICIES { + let policy = policies + .get(name) + .await + .map_err(|e| api_error("Verify SRE admission policy", e))?; + let binding = bindings + .get(name) + .await + .map_err(|e| api_error("Verify SRE admission binding", e))?; + let checked = policy.status.as_ref().is_some_and(|status| { + status.observed_generation == policy.metadata.generation + && status.type_checking.as_ref().is_some_and(|checking| { + checking + .expression_warnings + .as_ref() + .is_none_or(Vec::is_empty) + }) + }); + if policy.metadata.deletion_timestamp.is_some() + || binding.metadata.deletion_timestamp.is_some() + || policy + .spec + .as_ref() + .and_then(|spec| spec.failure_policy.as_deref()) + != Some("Fail") + || !checked + || binding.spec.as_ref().is_none_or(|spec| { + spec.policy_name.as_deref() != Some(*name) + || spec + .validation_actions + .as_ref() + .is_none_or(|actions| !actions.iter().any(|action| action == "Deny")) + }) + { + return Err(format!( + "SRE admission policy {name} is not observed, type-checked, and enforced; no privilege may be issued" + )); + } + } + Ok(()) +} diff --git a/controller/src/sre_authority/bindings.rs b/controller/src/sre_authority/bindings.rs new file mode 100644 index 000000000..36673f591 --- /dev/null +++ b/controller/src/sre_authority/bindings.rs @@ -0,0 +1,645 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::api_error; +use crate::sre_registration::{ + BindingReview, KarsSRERegistration, OWNER, ROUTER_SA, RUNTIME_NAMESPACE, +}; +use k8s_openapi::api::{ + core::v1::ServiceAccount, + rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, Role, RoleBinding, RoleRef, Subject}, +}; +use kube::{ + Api, Client, + api::{DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions}, +}; +use serde_json::json; + +const RETIRED: &str = "kars.azure.com/sre-legacy-retired"; +const GRANTS: &[(&str, &str)] = &[ + ("kars-sre-private-reader", "kars-sre-private-diagnostics"), + ("kars-sre-private-author", "kars-sre-action-author"), + ("kars-sre-private-renew", "kars-sre-router-renew"), +]; + +pub(crate) fn legacy_subject(subject: &Subject) -> bool { + (subject.kind == "ServiceAccount" + && subject.name == "sandbox" + && subject.namespace.as_deref() == Some(RUNTIME_NAMESPACE)) + || (subject.kind == "User" + && subject.name == format!("system:serviceaccount:{RUNTIME_NAMESPACE}:sandbox")) +} + +fn legacy_group(subject: &Subject) -> bool { + subject.kind == "Group" + && [ + "system:authenticated".to_string(), + "system:serviceaccounts".to_string(), + format!("system:serviceaccounts:{RUNTIME_NAMESPACE}"), + ] + .contains(&subject.name) +} + +fn dangerous(rules: &[PolicyRule]) -> bool { + rules.iter().any(|rule| { + let verbs = &rule.verbs; + let resources = rule.resources.as_deref().unwrap_or_default(); + let groups = rule.api_groups.as_deref().unwrap_or_default(); + let has = + |items: &[String], value: &str| items.iter().any(|item| item == "*" || item == value); + (has(groups, "") + && has(resources, "secrets") + && ["get", "list", "watch"].iter().any(|verb| has(verbs, verb))) + || (has(groups, "") + && ["pods/exec", "pods/proxy", "serviceaccounts/token"] + .iter() + .any(|resource| has(resources, resource)) + && (has(verbs, "create") || has(verbs, "get"))) + || (has(groups, "kars.azure.com") + && has(resources, "karssreactions") + && has(verbs, "create")) + }) +} + +async fn rules( + client: &Client, + role: &RoleRef, + namespace: Option<&str>, +) -> Result, String> { + if role.api_group != "rbac.authorization.k8s.io" { + return Err("Legacy binding roleRef uses an unsupported API group".into()); + } + match role.kind.as_str() { + "ClusterRole" => Ok(Api::::all(client.clone()) + .get(&role.name) + .await + .map_err(|e| api_error("Read legacy ClusterRole", e))? + .rules + .unwrap_or_default()), + "Role" => Ok(Api::::namespaced( + client.clone(), + namespace.ok_or("Role has no namespace")?, + ) + .get(&role.name) + .await + .map_err(|e| api_error("Read legacy Role", e))? + .rules + .unwrap_or_default()), + _ => Err("Legacy binding roleRef kind is invalid".into()), + } +} + +#[derive(Clone)] +pub(super) struct Binding { + review: BindingReview, + metadata: kube::api::ObjectMeta, + subjects: Vec, +} + +fn validate_review( + reg: &KarsSRERegistration, + kind: &str, + namespace: Option<&str>, + metadata: &kube::api::ObjectMeta, + role: &RoleRef, + subjects: &[Subject], +) -> Result { + let review = reg + .spec + .legacy_bindings + .iter() + .find(|review| { + review.kind == kind + && review.namespace.as_deref() == namespace + && Some(review.name.as_str()) == metadata.name.as_deref() + }) + .ok_or("Unreviewed legacy SRE binding exists; no migration mutations are authorized")?; + let retired = + metadata.annotations.as_ref().and_then(|a| a.get(RETIRED)) == reg.metadata.uid.as_ref(); + let expected: Vec<_> = review + .subjects + .iter() + .filter(|subject| !legacy_subject(subject)) + .cloned() + .collect(); + if metadata.uid.as_deref() != Some(review.uid.as_str()) + || metadata.deletion_timestamp.is_some() + || role != &review.role_ref + || (!retired + && (metadata.resource_version.as_deref() != Some(review.resource_version.as_str()) + || subjects != review.subjects)) + || (retired && subjects != expected) + { + return Err("Reviewed legacy grant changed or was replaced; migration stopped".into()); + } + Ok(Binding { + review: review.clone(), + metadata: metadata.clone(), + subjects: subjects.to_vec(), + }) +} + +pub(super) async fn review( + client: &Client, + reg: &KarsSRERegistration, +) -> Result, String> { + let cluster = Api::::all(client.clone()) + .list(&ListParams::default()) + .await + .map_err(|e| api_error("Inventory legacy ClusterRoleBindings", e))?; + let local = Api::::all(client.clone()) + .list(&ListParams::default()) + .await + .map_err(|e| api_error("Inventory legacy RoleBindings", e))?; + let mut bindings = Vec::new(); + let mut inspect = Vec::new(); + for binding in cluster { + inspect.push(( + "ClusterRoleBinding", + binding.metadata, + binding.role_ref, + binding.subjects.unwrap_or_default(), + )); + } + for binding in local { + inspect.push(( + "RoleBinding", + binding.metadata, + binding.role_ref, + binding.subjects.unwrap_or_default(), + )); + } + for (kind, metadata, role, subjects) in inspect { + if subjects.iter().any(legacy_group) + && dangerous(&rules(client, &role, metadata.namespace.as_deref()).await?) + { + return Err("A broad group grant gives legacy SRE credentials privileged access; restructure that grant explicitly".into()); + } + let was_reviewed = reg.spec.legacy_bindings.iter().any(|review| { + review.kind == kind + && review.name == metadata.name.as_deref().unwrap_or_default() + && review.namespace == metadata.namespace + }); + let legacy = subjects.iter().any(legacy_subject); + let ordinary_spawner = legacy + && role.kind == "ClusterRole" + && role.name == "kars-sandbox-spawner" + && rules(client, &role, metadata.namespace.as_deref()) + .await? + .iter() + .all(|rule| { + rule.api_groups.as_deref() == Some(&["kars.azure.com".into()][..]) + && rule.resources.as_deref() == Some(&["karssandboxes".into()][..]) + && rule.non_resource_urls.is_none() + && rule.verbs.iter().all(|verb| { + ["get", "list", "create", "delete"].contains(&verb.as_str()) + }) + }); + if (legacy && !ordinary_spawner) || was_reviewed { + bindings.push(validate_review( + reg, + kind, + metadata.namespace.as_deref(), + &metadata, + &role, + &subjects, + )?); + } + } + Ok(bindings) +} + +pub(super) async fn retire_legacy( + client: &Client, + reg: &KarsSRERegistration, + bindings: &[Binding], +) -> Result<(), String> { + let plans: Vec<_> = bindings + .iter() + .filter(|binding| binding.subjects.iter().any(legacy_subject)) + .map(|binding| { + let subjects: Vec<_> = binding + .subjects + .iter() + .filter(|subject| !legacy_subject(subject)) + .cloned() + .collect(); + let patch = json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version, + "annotations":{RETIRED:reg.metadata.uid}}, + "subjects":subjects, + }); + (binding, patch) + }) + .collect(); + // RBAC escalation checks also apply to subtractive binding updates. + // Preflight every exact patch before any write; this is not a transaction. + for dry_run in [true, false] { + let params = PatchParams { + dry_run, + ..Default::default() + }; + for (binding, patch) in &plans { + if binding.review.kind == "ClusterRoleBinding" { + Api::::all(client.clone()) + .patch(&binding.review.name, ¶ms, &Patch::Merge(patch)) + .await + .map_err(|e| { + api_error( + if dry_run { + "Preflight reviewed SRE ClusterRoleBinding retirement" + } else { + "Retire reviewed SRE ClusterRoleBinding" + }, + e, + ) + })?; + } else { + Api::::namespaced( + client.clone(), + binding.review.namespace.as_deref().unwrap(), + ) + .patch(&binding.review.name, ¶ms, &Patch::Merge(patch)) + .await + .map_err(|e| { + api_error( + if dry_run { + "Preflight reviewed SRE RoleBinding retirement" + } else { + "Retire reviewed SRE RoleBinding" + }, + e, + ) + })?; + } + } + } + Ok(()) +} + +fn owned(meta: &kube::api::ObjectMeta, reg: &KarsSRERegistration) -> bool { + meta.annotations.as_ref().is_some_and(|a| { + a.get(OWNER) == reg.metadata.uid.as_ref() + && a.get("kars.azure.com/namespace-uid") == Some(®.spec.runtime_namespace.uid) + }) && meta.uid.as_deref().is_some_and(|uid| !uid.is_empty()) + && meta + .resource_version + .as_deref() + .is_some_and(|rv| !rv.is_empty()) + && meta.deletion_timestamp.is_none() +} + +pub(super) async fn validate_private_targets( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let bindings: Api = Api::all(client.clone()); + for (name, role) in GRANTS { + let definition = Api::::all(client.clone()) + .get(role) + .await + .map_err(|e| api_error("Preflight private SRE role", e))?; + if !safe_private_role(role, definition.rules.as_deref().unwrap_or_default()) { + return Err( + "Private SRE role definition has excessive or unsupported authority".into(), + ); + } + if let Some(binding) = bindings + .get_opt(name) + .await + .map_err(|e| api_error("Preflight private SRE grant", e))? + && (!owned(&binding.metadata, reg) + || binding.role_ref.name != *role + || binding.role_ref.kind != "ClusterRole" + || binding.role_ref.api_group != "rbac.authorization.k8s.io" + || binding.subjects.as_deref().is_none_or(|subjects| { + subjects.len() != 1 + || subjects[0].kind != "ServiceAccount" + || subjects[0].name != ROUTER_SA + || subjects[0].namespace.as_deref() != Some(RUNTIME_NAMESPACE) + })) + { + return Err( + "Reserved private SRE binding already has another owner or authority".into(), + ); + } + } + let roles: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(role) = roles + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Preflight renewal Role", e))? + && (!owned(&role.metadata, reg) + || serde_json::to_value(&role.rules).ok() + != Some(json!([{ + "apiGroups":[""],"resources":["serviceaccounts/token"],"resourceNames":[ROUTER_SA],"verbs":["create"], + }]))) + { + return Err( + "Reserved renewal Role belongs to another owner or has different authority".into(), + ); + } + let bindings: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(binding) = bindings + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Preflight renewal binding", e))? + && (!owned(&binding.metadata, reg) + || binding.role_ref.kind != "Role" + || binding.role_ref.name != "sre-api-self-renew" + || binding.subjects.as_deref().is_none_or(|subjects| { + subjects.len() != 1 + || subjects[0].kind != "ServiceAccount" + || subjects[0].name != ROUTER_SA + || subjects[0].namespace.as_deref() != Some(RUNTIME_NAMESPACE) + })) + { + return Err( + "Reserved renewal binding belongs to another owner or has different authority".into(), + ); + } + Ok(()) +} + +pub(super) async fn grant_private( + client: &Client, + reg: &KarsSRERegistration, + sa: &ServiceAccount, +) -> Result<(), String> { + super::credential_guard::scan(client, reg).await?; + if !owned(&sa.metadata, reg) { + return Err("Private SRE ServiceAccount ownership is not proven".into()); + } + let subjects = vec![Subject { + kind: "ServiceAccount".into(), + name: ROUTER_SA.into(), + namespace: Some(RUNTIME_NAMESPACE.into()), + api_group: None, + }]; + let api: Api = Api::all(client.clone()); + for (name, role) in GRANTS { + let definition = Api::::all(client.clone()) + .get(role) + .await + .map_err(|e| api_error("Validate private SRE role definition", e))?; + if !safe_private_role(role, definition.rules.as_deref().unwrap_or_default()) { + return Err( + "Private SRE role definition contains unreviewed or excessive authority".into(), + ); + } + super::live::verify(client, reg).await?; + let body: ClusterRoleBinding = serde_json::from_value(json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":name,"annotations":{OWNER:reg.metadata.uid, + "kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":role}, + "subjects":subjects, + })) + .map_err(|_| "Private SRE binding serialization failed")?; + match api + .get_opt(name) + .await + .map_err(|e| api_error("Read private SRE binding", e))? + { + None => { + api.create(&PostParams::default(), &body) + .await + .map_err(|e| api_error("Create private SRE binding", e))?; + } + Some(existing) => { + if !owned(&existing.metadata, reg) + || existing.role_ref != body.role_ref + || existing.subjects != body.subjects + { + return Err("Private SRE binding conflicts with an existing resource".into()); + } + } + } + } + let roles: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let role: Role = serde_json::from_value(json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"Role", + "metadata":{"name":"sre-api-self-renew","namespace":RUNTIME_NAMESPACE,"annotations":{ + OWNER:reg.metadata.uid,"kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}}, + "rules":[{"apiGroups":[""],"resources":["serviceaccounts/token"],"resourceNames":[ROUTER_SA],"verbs":["create"]}], + })).map_err(|_| "SRE token renewal Role serialization failed")?; + match roles + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read renewal Role", e))? + { + None => { + roles + .create(&PostParams::default(), &role) + .await + .map_err(|e| api_error("Create renewal Role", e))?; + } + Some(existing) if owned(&existing.metadata, reg) && existing.rules == role.rules => {} + Some(_) => { + return Err( + "Existing SRE token renewal Role is not owned or has different authority".into(), + ); + } + } + let bindings: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let binding: RoleBinding = serde_json::from_value(json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBinding", + "metadata":{"name":"sre-api-self-renew","namespace":RUNTIME_NAMESPACE,"annotations":{ + OWNER:reg.metadata.uid,"kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"sre-api-self-renew"}, + "subjects":subjects, + })).map_err(|_| "SRE renewal binding serialization failed")?; + match bindings + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read renewal binding", e))? + { + None => { + bindings + .create(&PostParams::default(), &binding) + .await + .map_err(|e| api_error("Create renewal binding", e))?; + } + Some(existing) + if owned(&existing.metadata, reg) + && existing.subjects == binding.subjects + && existing.role_ref == binding.role_ref => {} + Some(_) => { + return Err( + "Existing SRE renewal binding is not owned or has different authority".into(), + ); + } + } + Ok(()) +} + +fn safe_private_role(name: &str, rules: &[PolicyRule]) -> bool { + !rules.is_empty() + && rules.iter().all(|rule| { + let resources = rule.resources.as_deref().unwrap_or_default(); + let groups = rule.api_groups.as_deref().unwrap_or_default(); + if resources.is_empty() + || resources.iter().any(|r| { + r == "*" + || ["/proxy", "/exec", "/attach", "/portforward", "/token"] + .iter() + .any(|suffix| r.ends_with(suffix)) + }) + { + return false; + } + match name { + "kars-sre-private-diagnostics" => rule + .verbs + .iter() + .all(|verb| ["get", "list", "watch"].contains(&verb.as_str())), + "kars-sre-action-author" => { + groups == ["kars.azure.com"] + && resources.iter().all(|r| { + ["karssreactions", "karssreactions/status"].contains(&r.as_str()) + }) + && rule + .verbs + .iter() + .all(|verb| ["get", "list", "watch", "create"].contains(&verb.as_str())) + } + "kars-sre-router-renew" => { + (groups == ["kars.azure.com"] + && resources == ["karssreregistrations"] + && rule.resource_names.as_deref() == Some(&["canonical".into()][..]) + && rule + .verbs + .iter() + .all(|verb| ["get", "renew"].contains(&verb.as_str()))) + || (groups == ["authorization.k8s.io"] + && resources == ["subjectaccessreviews"] + && rule.verbs == ["create"] + && rule.non_resource_urls.is_none()) + } + _ => false, + } + }) +} + +pub(super) async fn revoke_private( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let private_subject = |subject: &&Subject| { + (subject.kind == "ServiceAccount" + && subject.name == ROUTER_SA + && subject.namespace.as_deref() == Some(RUNTIME_NAMESPACE)) + || (subject.kind == "User" + && subject.name == format!("system:serviceaccount:{RUNTIME_NAMESPACE}:{ROUTER_SA}")) + }; + let mut foreign = false; + let api: Api = Api::all(client.clone()); + for (name, _) in GRANTS { + if let Some(binding) = api + .get_opt(name) + .await + .map_err(|e| api_error("Read retiring SRE binding", e))? + { + if !owned(&binding.metadata, reg) { + foreign = true; + continue; + } + let retained: Vec<_> = binding + .subjects + .as_deref() + .unwrap_or_default() + .iter() + .filter(|subject| !private_subject(subject)) + .cloned() + .collect(); + if !retained.is_empty() { + api.patch(name, &PatchParams::default(), &Patch::Merge(json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version}, + "subjects":retained, + }))).await.map_err(|e| api_error("Revoke only the owned private SRE subject", e))?; + continue; + } + api.delete( + name, + &DeleteParams { + preconditions: Some(Preconditions { + uid: binding.metadata.uid, + resource_version: binding.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire private SRE binding", e))?; + } + } + let mut retain_role = false; + let bindings: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(binding) = bindings + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read retiring SRE renewal binding", e))? + { + if !owned(&binding.metadata, reg) { + return Err("Refusing to retire an unowned renewal binding".into()); + } + let retained: Vec<_> = binding + .subjects + .as_deref() + .unwrap_or_default() + .iter() + .filter(|subject| !private_subject(subject)) + .cloned() + .collect(); + if !retained.is_empty() { + bindings.patch("sre-api-self-renew", &PatchParams::default(), &Patch::Merge(json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version}, + "subjects":retained, + }))).await.map_err(|e| api_error("Revoke only the private SRE renewal subject", e))?; + retain_role = true; + } else { + bindings + .delete( + "sre-api-self-renew", + &DeleteParams { + preconditions: Some(Preconditions { + uid: binding.metadata.uid, + resource_version: binding.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire SRE renewal binding", e))?; + } + } + let roles: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(role) = roles + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read retiring SRE renewal Role", e))? + && !retain_role + { + if !owned(&role.metadata, reg) { + return Err("Refusing to retire an unowned renewal Role".into()); + } + roles + .delete( + "sre-api-self-renew", + &DeleteParams { + preconditions: Some(Preconditions { + uid: role.metadata.uid, + resource_version: role.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire SRE renewal Role", e))?; + } + if foreign { + Err("Unowned private SRE bindings were preserved; owned grants were revoked".into()) + } else { + Ok(()) + } +} diff --git a/controller/src/sre_authority/credential_guard.rs b/controller/src/sre_authority/credential_guard.rs new file mode 100644 index 000000000..93d23d904 --- /dev/null +++ b/controller/src/sre_authority/credential_guard.rs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::api_error; +use crate::sre_registration::{KarsSRERegistration, ROUTER_SA, RUNTIME_NAMESPACE}; +use k8s_openapi::api::core::v1::{Secret, ServiceAccount}; +use kube::{Api, Client, api::ListParams}; + +pub(super) async fn scan(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + let account = Api::::namespaced(client.clone(), RUNTIME_NAMESPACE) + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Inspect reserved SRE token identity", e))?; + let mut uids = Vec::new(); + if let Some(uid) = account.as_ref().and_then(|sa| sa.metadata.uid.as_deref()) { + uids.push(uid); + } + if let Some(uid) = reg + .status + .as_ref() + .and_then(|status| status.router_service_account_uid.as_deref()) + { + uids.push(uid); + } + let metadata = Api::::namespaced(client.clone(), RUNTIME_NAMESPACE) + .list_metadata(&ListParams::default()) + .await + .map_err(|e| api_error("Inspect prestaged SRE token Secret metadata", e))?; + let list = serde_json::to_value(metadata) + .map_err(|_| "SRE token metadata inventory could not be decoded")?; + crate::sre_privacy::reject_legacy_aliases(&list, &uids).map_err(str::to_owned) +} diff --git a/controller/src/sre_authority/credentials.rs b/controller/src/sre_authority/credentials.rs new file mode 100644 index 000000000..3213c71e5 --- /dev/null +++ b/controller/src/sre_authority/credentials.rs @@ -0,0 +1,423 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::{api_error, live::Authority}; +use crate::sre_registration::{ + AGENT_SECRET, EPOCH, KarsSRERegistration, OWNER, PRIVATE_SECRET, ROUTER_SA, RUNTIME_NAMESPACE, +}; +use k8s_openapi::api::{ + authentication::v1::TokenRequest, + core::v1::{Secret, ServiceAccount}, +}; +use kube::{ + Api, Client, ResourceExt, + api::{DeleteParams, Patch, PatchParams, PostParams, Preconditions}, +}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +fn annotations(reg: &KarsSRERegistration) -> Value { + json!({OWNER:reg.metadata.uid,EPOCH:reg.epoch(), + "kars.azure.com/sandbox-uid":reg.spec.sandbox.uid, + "kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}) +} + +fn owned(meta: &kube::api::ObjectMeta, reg: &KarsSRERegistration) -> bool { + meta.namespace.as_deref() == Some(RUNTIME_NAMESPACE) + && meta.annotations.as_ref().is_some_and(|a| { + a.get(OWNER) == reg.metadata.uid.as_ref() + && a.get("kars.azure.com/sandbox-uid") == Some(®.spec.sandbox.uid) + && a.get("kars.azure.com/namespace-uid") == Some(®.spec.runtime_namespace.uid) + }) + && meta.uid.as_deref().is_some_and(|uid| !uid.is_empty()) + && meta + .resource_version + .as_deref() + .is_some_and(|rv| !rv.is_empty()) + && meta.deletion_timestamp.is_none() +} + +fn data(secret: &Secret, key: &str) -> Option { + String::from_utf8(secret.data.as_ref()?.get(key)?.0.clone()).ok() +} + +pub(super) async fn review_targets( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + super::credential_guard::scan(client, reg).await?; + let accounts: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(sa) = accounts + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Preflight private SRE identity", e))? + && (!owned(&sa.metadata, reg) || sa.automount_service_account_token != Some(false)) + { + return Err("Reserved SRE ServiceAccount already belongs to another owner".into()); + } + let secrets: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + for name in [PRIVATE_SECRET, AGENT_SECRET] { + if let Some(secret) = secrets + .get_metadata_opt(name) + .await + .map_err(|e| api_error("Preflight SRE material ownership", e))? + && !owned(&secret.metadata, reg) + { + return Err("Reserved SRE identity Secret already belongs to another owner".into()); + } + } + Ok(()) +} + +pub(super) async fn ensure_service_account( + client: &Client, + reg: &KarsSRERegistration, + authority: &Authority, +) -> Result { + super::credential_guard::scan(client, reg).await?; + if authority.namespace.metadata.uid.as_deref() != Some(®.spec.runtime_namespace.uid) + || authority.sandbox.metadata.uid.as_deref() != Some(®.spec.sandbox.uid) + { + return Err("SRE authority changed before private identity creation".into()); + } + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(sa) = api + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Read private SRE ServiceAccount", e))? + { + if !owned(&sa.metadata, reg) + || sa.automount_service_account_token != Some(false) + || reg + .status + .as_ref() + .and_then(|s| s.router_service_account_uid.as_deref()) + .is_some_and(|uid| Some(uid) != sa.metadata.uid.as_deref()) + { + return Err("Private SRE ServiceAccount is unowned, replaced, or unsafe".into()); + } + return Ok(sa); + } + let sa: ServiceAccount = serde_json::from_value(json!({ + "apiVersion":"v1","kind":"ServiceAccount","metadata":{ + "name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE,"annotations":annotations(reg)}, + "automountServiceAccountToken":false, + })) + .map_err(|_| "Private SRE ServiceAccount serialization failed")?; + api.create(&PostParams::default(), &sa) + .await + .map_err(|e| api_error("Create private SRE ServiceAccount", e)) +} + +async fn secret(client: &Client, reg: &KarsSRERegistration, name: &str) -> Result { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(secret) = api + .get_opt(name) + .await + .map_err(|e| api_error("Read owned SRE identity material", e))? + { + if !owned(&secret.metadata, reg) { + return Err( + "Existing SRE identity material has a different owner or namespace incarnation" + .into(), + ); + } + return Ok(secret); + } + let empty: Secret = serde_json::from_value(json!({ + "apiVersion":"v1","kind":"Secret","type":"Opaque", + "metadata":{"name":name,"namespace":RUNTIME_NAMESPACE,"annotations":annotations(reg)}, + })) + .map_err(|_| "SRE identity metadata serialization failed")?; + api.create(&PostParams::default(), &empty) + .await + .map_err(|e| api_error("Create owned SRE identity metadata", e)) +} + +async fn update( + client: &Client, + secret: &Secret, + values: BTreeMap, + annotations: Value, +) -> Result<(), String> { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + api.patch(&secret.name_any(),&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":secret.metadata.uid,"resourceVersion":secret.metadata.resource_version,"annotations":annotations}, + "type":"Opaque","stringData":values, + }))).await.map_err(|e|api_error("Update owned SRE identity material",e))?; + Ok(()) +} + +fn cluster_connection() -> Result<(String, String), String> { + let host = std::env::var("KUBERNETES_SERVICE_HOST") + .map_err(|_| "Kubernetes service host is unavailable")?; + let port = std::env::var("KUBERNETES_SERVICE_PORT_HTTPS").unwrap_or_else(|_| "443".into()); + let host = if host.contains(':') { + format!("[{host}]") + } else { + host + }; + let url = reqwest::Url::parse(&format!("https://{host}:{port}")) + .map_err(|_| "Kubernetes service address is invalid")?; + let ca = std::fs::read_to_string("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + .map_err(|_| "Kubernetes service CA is unavailable")?; + Ok((url.to_string(), ca)) +} + +pub(super) async fn ensure( + client: &Client, + reg: &KarsSRERegistration, + authority: &Authority, + sa: &ServiceAccount, +) -> Result<(), String> { + ensure_with_connection(client, reg, authority, sa, cluster_connection).await +} + +async fn ensure_with_connection( + client: &Client, + reg: &KarsSRERegistration, + _authority: &Authority, + sa: &ServiceAccount, + connection: impl FnOnce() -> Result<(String, String), String>, +) -> Result<(), String> { + super::live::verify(client, reg).await?; + super::check_secret_denial(client, RUNTIME_NAMESPACE).await?; + super::credential_guard::scan(client, reg).await?; + let mut private = secret(client, reg, PRIVATE_SECRET).await?; + if private + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(EPOCH)) + != Some(®.epoch()) + || (reg + .status + .as_ref() + .is_some_and(|status| status.phase == "Blocked") + && data(&private, "kube-token").is_some()) + { + // Replacing the bound Secret UID invalidates old private JWTs, not just + // the current file contents, when upgrading an underguarded epoch. + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + api.delete( + PRIVATE_SECRET, + &DeleteParams { + preconditions: Some(Preconditions { + uid: private.metadata.uid.clone(), + resource_version: private.metadata.resource_version.clone(), + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire prior-epoch SRE token anchor", e))?; + private = secret(client, reg, PRIVATE_SECRET).await?; + } + let agent = secret(client, reg, AGENT_SECRET).await?; + let mut private_values = BTreeMap::new(); + let now = chrono::Utc::now().timestamp(); + let epoch = reg.epoch(); + let tls_expiry = private + .metadata + .annotations + .as_ref() + .and_then(|a| a.get("kars.azure.com/sre-tls-expiry")) + .and_then(|value| value.parse::().ok()) + .unwrap_or_default(); + let same_epoch = private + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(EPOCH)) + == Some(&epoch); + let tls_valid = same_epoch + && tls_expiry > now + 172_800 + && [ + "server-cert.pem", + "server-key.pem", + "agent-token", + "agent-ca.crt", + ] + .iter() + .all(|key| data(&private, key).is_some()); + let mut private_annotations = annotations(reg); + if !tls_valid { + let identity = crate::providers::sre_tls::issue()?; + let proxy_token = crate::providers::signing::generate_service_token(); + private_values.insert("server-cert.pem".into(), identity.certificate); + private_values.insert("server-key.pem".into(), identity.private_key); + private_values.insert("agent-ca.crt".into(), identity.ca.clone()); + private_values.insert("agent-token".into(), proxy_token.clone()); + private_annotations["kars.azure.com/sre-tls-expiry"] = + identity.expires_at.to_string().into(); + // Token and CA change together. The legacy Hermes client rebuilds its + // TLS context when it observes the opaque token change. + update( + client, + &agent, + BTreeMap::from([ + ("token".into(), proxy_token), + ("ca.crt".into(), identity.ca), + ("namespace".into(), RUNTIME_NAMESPACE.into()), + ]), + annotations(reg), + ) + .await?; + } else if data(&agent, "token") != data(&private, "agent-token") + || data(&agent, "ca.crt") != data(&private, "agent-ca.crt") + { + update( + client, + &agent, + BTreeMap::from([ + ( + "token".into(), + data(&private, "agent-token").ok_or("SRE proxy token missing")?, + ), + ( + "ca.crt".into(), + data(&private, "agent-ca.crt").ok_or("SRE proxy CA missing")?, + ), + ("namespace".into(), RUNTIME_NAMESPACE.into()), + ]), + annotations(reg), + ) + .await?; + } + let kube_expiry = data(&private, "kube-expires-at") + .and_then(|value| chrono::DateTime::parse_from_rfc3339(&value).ok()) + .map(|time| time.timestamp()) + .unwrap_or_default(); + if !same_epoch || kube_expiry < now + 600 { + let request: TokenRequest = serde_json::from_value(json!({ + "apiVersion":"authentication.k8s.io/v1","kind":"TokenRequest", + "spec":{"audiences":[],"expirationSeconds":3600, + "boundObjectRef":{"apiVersion":"v1","kind":"Secret","name":PRIVATE_SECRET,"uid":private.metadata.uid}}, + })).map_err(|_|"SRE TokenRequest serialization failed")?; + let token = Api::::namespaced(client.clone(), RUNTIME_NAMESPACE) + .create_token_request(ROUTER_SA, &PostParams::default(), &request) + .await + .map_err(|e| api_error("Issue private SRE Kubernetes token", e))?; + let token = token + .status + .ok_or("Kubernetes TokenRequest omitted its token/expiry")?; + let expiry = token.expiration_timestamp.0.to_string(); + if token.token.is_empty() + || chrono::DateTime::parse_from_rfc3339(&expiry) + .map_err(|_| "TokenRequest expiry was invalid")? + .timestamp() + <= now + 300 + { + return Err("TokenRequest returned unusable private credentials".into()); + } + let (url, ca) = connection()?; + private_values.insert("kube-token".into(), token.token); + private_values.insert("kube-expires-at".into(), expiry); + private_values.insert("kube-ca.crt".into(), ca); + private_values.insert( + "config.json".into(), + serde_json::to_string(&json!({ + "schema":"kars.azure.com/sre-api/v1","kubeUrl":url, + "registrationUid":reg.metadata.uid,"privacyEpoch":epoch, + "source":reg.spec.sandbox,"namespaceUid":reg.spec.runtime_namespace.uid, + "runtimeNamespace":RUNTIME_NAMESPACE,"serviceAccountUid":sa.metadata.uid, + "secretUid":private.metadata.uid, + })) + .map_err(|_| "SRE proxy configuration serialization failed")?, + ); + } + + if !private_values.is_empty() { + super::live::verify(client, reg).await?; + super::check_secret_denial(client, RUNTIME_NAMESPACE).await?; + super::credential_guard::scan(client, reg).await?; + update(client, &private, private_values, private_annotations).await?; + } + Ok(()) +} + +#[cfg(test)] +pub(super) async fn ensure_for_test( + client: &Client, + reg: &KarsSRERegistration, + authority: &Authority, + sa: &ServiceAccount, +) -> Result<(), String> { + ensure_with_connection(client, reg, authority, sa, || { + Ok(("https://kubernetes.default.svc".into(), "test-ca".into())) + }) + .await +} + +pub(super) async fn retire(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + let secrets: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + for name in [PRIVATE_SECRET, AGENT_SECRET] { + if let Some(secret) = secrets + .get_opt(name) + .await + .map_err(|e| api_error("Read retiring SRE identity", e))? + { + if secret + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(OWNER)) + != reg.metadata.uid.as_ref() + { + continue; + } + if !owned(&secret.metadata, reg) { + return Err("SRE identity material has conflicting incarnation metadata".into()); + } + secrets + .delete( + name, + &DeleteParams { + preconditions: Some(Preconditions { + uid: secret.metadata.uid, + resource_version: secret.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire owned SRE identity", e))?; + } + } + let accounts: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(sa) = accounts + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Read retiring private SRE identity", e))? + { + if sa.metadata.annotations.as_ref().and_then(|a| a.get(OWNER)) != reg.metadata.uid.as_ref() + { + return Ok(()); + } + if !owned(&sa.metadata, reg) { + return Err("Private SRE identity has conflicting incarnation metadata".into()); + } + if reg + .status + .as_ref() + .and_then(|status| status.router_service_account_uid.as_deref()) + .is_some_and(|uid| sa.metadata.uid.as_deref() != Some(uid)) + { + return Err("Private SRE ServiceAccount UID changed; replacement preserved".into()); + } + accounts + .delete( + ROUTER_SA, + &DeleteParams { + preconditions: Some(Preconditions { + uid: sa.metadata.uid, + resource_version: sa.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire private SRE identity", e))?; + } + Ok(()) +} diff --git a/controller/src/sre_authority/live.rs b/controller/src/sre_authority/live.rs new file mode 100644 index 000000000..62a20bf36 --- /dev/null +++ b/controller/src/sre_authority/live.rs @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + crd::KarsSandbox, + sre_registration::{KarsSRERegistration, NAME}, +}; +use k8s_openapi::api::{ + apps::v1::Deployment, authorization::v1::SubjectAccessReview, core::v1::Namespace, +}; +use kube::{Api, Client, api::PostParams}; + +pub(super) struct Authority { + pub sandbox: KarsSandbox, + pub namespace: Namespace, +} + +pub(crate) fn api_error(stage: &str, error: kube::Error) -> String { + match error { + kube::Error::Api(status) => format!("{stage}: Kubernetes status {}", status.code), + _ => format!("{stage}: Kubernetes transport/serialization failure"), + } +} + +pub(super) async fn verify( + client: &Client, + reg: &KarsSRERegistration, +) -> Result { + reg.validate()?; + let registrations: Api = Api::all(client.clone()); + let current = registrations + .get(NAME) + .await + .map_err(|e| api_error("Read live SRE registration", e))?; + if current.metadata.uid != reg.metadata.uid + || current.metadata.generation != reg.metadata.generation + || current.metadata.deletion_timestamp.is_some() + { + return Err("SRE registration changed during reconciliation".into()); + } + let namespaces: Api = Api::all(client.clone()); + let control_ns = namespaces + .get(®.spec.controller.namespace.name) + .await + .map_err(|e| api_error("Read registered controller namespace", e))?; + if control_ns.metadata.uid.as_deref() != Some(®.spec.controller.namespace.uid) + || control_ns.metadata.deletion_timestamp.is_some() + { + return Err("Registered controller namespace was replaced or is terminating".into()); + } + let deployments: Api = + Api::namespaced(client.clone(), ®.spec.controller.namespace.name); + let controller = deployments + .get(®.spec.controller.deployment.name) + .await + .map_err(|e| api_error("Read registered controller Deployment", e))?; + if controller.metadata.uid.as_deref() != Some(®.spec.controller.deployment.uid) + || controller.metadata.deletion_timestamp.is_some() + || controller + .metadata + .annotations + .as_ref() + .is_some_and(|annotations| { + annotations + .get("meta.helm.sh/release-name") + .is_some_and(|name| name != ®.spec.controller.release) + || annotations + .get("meta.helm.sh/release-namespace") + .is_some_and(|name| name != ®.spec.controller.namespace.name) + }) + { + return Err("Registered controller/release identity does not match".into()); + } + let sandboxes: Api = Api::namespaced(client.clone(), ®.spec.sandbox.namespace); + let sandbox = sandboxes + .get(®.spec.sandbox.name) + .await + .map_err(|e| api_error("Read registered SRE Sandbox", e))?; + let namespace = namespaces + .get(®.spec.runtime_namespace.name) + .await + .map_err(|e| api_error("Read registered SRE runtime namespace", e))?; + if sandbox.metadata.uid.as_deref() != Some(®.spec.sandbox.uid) + || sandbox.metadata.deletion_timestamp.is_some() + || sandbox + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get("kars.azure.com/role")) + .map(String::as_str) + != Some("sre") + || namespace.metadata.uid.as_deref() != Some(®.spec.runtime_namespace.uid) + || namespace.metadata.deletion_timestamp.is_some() + || sandbox + .metadata + .annotations + .as_ref() + .and_then(|a| a.get("kars.azure.com/namespace-uid")) + != Some(®.spec.runtime_namespace.uid) + || !crate::reconciler::namespace_ownership::claimed(&namespace, &sandbox) + .map_err(|_| "Registered SRE namespace claim is invalid")? + { + return Err( + "Registered SRE source/runtime UID or claim does not match; foreign occupant preserved" + .into(), + ); + } + Ok(Authority { sandbox, namespace }) +} + +pub(crate) async fn check_secret_denial(client: &Client, namespace: &str) -> Result<(), String> { + let reviews: Api = Api::all(client.clone()); + for request in crate::sre_privacy::secret_access_reviews(namespace) { + let review: SubjectAccessReview = serde_json::from_value(request) + .map_err(|_| "SRE Secret authorization request is invalid")?; + let checked = reviews + .create(&PostParams::default(), &review) + .await + .map_err(|e| api_error("Verify legacy SRE Secret denial", e))?; + let response = serde_json::to_value(checked) + .map_err(|_| "SRE Secret authorization response is invalid")?; + crate::sre_privacy::require_denial(&response)?; + } + Ok(()) +} + +/// Later private-credential issuers must call this immediately before issuance. +/// An absent registration is safe only when the old SRE subject has no access. +pub(crate) async fn privacy_epoch( + client: &Client, + target_namespace: &str, +) -> Result, String> { + check_secret_denial(client, target_namespace).await?; + let registrations: Api = Api::all(client.clone()); + let Some(reg) = registrations + .get_opt(NAME) + .await + .map_err(|e| api_error("Read SRE privacy epoch", e))? + else { + return Ok(None); + }; + if !reg.spec.enabled + && reg.status.as_ref().is_some_and(|status| { + status.phase == "Retired" + && status.observed_generation == reg.metadata.generation.unwrap_or_default() + }) + { + return Ok(None); + } + verify(client, ®).await?; + super::admission::verify(client).await?; + super::credential_guard::scan(client, ®).await?; + let status = reg + .status + .as_ref() + .ok_or("SRE authority has not been reconciled")?; + if status.phase != "Ready" + || status.observed_generation != reg.metadata.generation.unwrap_or_default() + || !status.legacy_secret_access_denied + || status.privacy_revision.as_deref() != Some(crate::sre_privacy::REVISION) + || status.privacy_epoch.as_deref() != Some(reg.epoch().as_str()) + { + return Err("SRE privacy migration is not Ready".into()); + } + Ok(status.privacy_epoch.clone()) +} diff --git a/controller/src/sre_authority/migration.rs b/controller/src/sre_authority/migration.rs new file mode 100644 index 000000000..79e2a4dc1 --- /dev/null +++ b/controller/src/sre_authority/migration.rs @@ -0,0 +1,431 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::{api_error, check_secret_denial}; +use crate::{ + crd::KarsSandbox, + sre_registration::{EPOCH, KarsSRERegistration, OWNER, RUNTIME_NAMESPACE}, +}; +use k8s_openapi::api::{ + apps::v1::Deployment, + core::v1::{Namespace, Pod, Secret}, +}; +use kube::{ + Api, Client, ResourceExt, + api::{DeleteParams, ListParams, Patch, PatchParams, Preconditions}, +}; +use serde_json::json; + +const STOPPED: &str = "kars.azure.com/sre-migration-stopped"; +pub(super) const WAITING_FOR_CONSUMERS: &str = + "Waiting for legacy SRE consumers to terminate; no private credentials issued"; +const WAITING_FOR_ROTATION: &str = + "Waiting for owned control credential consumers to restart on the new privacy epoch"; +const WAITING_FOR_ROLLOUT: &str = + "Owned control credential consumer has not completed its privacy-epoch rollout"; +const WAITING_FOR_CONSUMER_REMOVAL: &str = + "Waiting for the owned SRE consumer Deployment to be removed"; + +pub(super) fn is_waiting(detail: &str) -> bool { + matches!( + detail, + WAITING_FOR_CONSUMERS + | WAITING_FOR_ROTATION + | WAITING_FOR_ROLLOUT + | WAITING_FOR_CONSUMER_REMOVAL + ) +} + +pub(super) async fn stop_registered_consumer_for_retirement( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let namespaces: Api = Api::all(client.clone()); + let Some(namespace) = namespaces + .get_opt(RUNTIME_NAMESPACE) + .await + .map_err(|e| api_error("Read retiring SRE namespace", e))? + else { + return Ok(()); + }; + if namespace.metadata.uid.as_deref() != Some(reg.spec.runtime_namespace.uid.as_str()) { + return Ok(()); + } + let registration_uid = reg + .metadata + .uid + .as_deref() + .filter(|uid| !uid.is_empty()) + .ok_or("SRE retirement requires the actual registration UID")?; + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let Some(deployment) = api + .get_opt("sre") + .await + .map_err(|e| api_error("Read retiring SRE consumer", e))? + else { + return Ok(()); + }; + let ours = deployment + .spec + .as_ref() + .and_then(|s| s.template.metadata.as_ref()) + .and_then(|m| m.annotations.as_ref()) + .and_then(|a| a.get(OWNER)) + .map(String::as_str) + == Some(registration_uid); + if !ours + && reg + .status + .as_ref() + .and_then(|s| s.router_service_account_uid.as_ref()) + .is_none() + { + return Ok(()); + } + stop_legacy_consumer(client, reg).await?; + if !ours { + return Ok(()); + } + // Namespace-controller intentionally lacks registrar power. Remove only + // our stopped Deployment here instead of leaving protected content to GC. + let Some(current) = api + .get_opt("sre") + .await + .map_err(|error| api_error("Read stopped SRE consumer", error))? + else { + return Ok(()); + }; + let uid = deployment + .metadata + .uid + .as_deref() + .filter(|uid| !uid.is_empty()) + .ok_or("Retiring SRE consumer UID is missing")?; + let still_owned = current + .spec + .as_ref() + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|metadata| metadata.annotations.as_ref()) + .and_then(|annotations| annotations.get(OWNER)) + .map(String::as_str) + == Some(registration_uid); + if current.metadata.uid.as_deref() != Some(uid) + || !still_owned + || current.spec.as_ref().and_then(|spec| spec.replicas) != Some(0) + { + return Err("Retiring SRE consumer changed after quiescence; preserved".into()); + } + if current.metadata.deletion_timestamp.is_some() { + return Err(WAITING_FOR_CONSUMER_REMOVAL.into()); + } + let version = current + .metadata + .resource_version + .filter(|version| !version.is_empty()) + .ok_or("Retiring SRE consumer resourceVersion is missing")?; + match api + .delete( + "sre", + &DeleteParams { + preconditions: Some(Preconditions { + uid: Some(uid.into()), + resource_version: Some(version), + }), + ..Default::default() + }, + ) + .await + { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(api_error("Remove stopped owned SRE consumer", error)), + } + if api + .get_opt("sre") + .await + .map_err(|error| api_error("Verify owned SRE consumer removal", error))? + .is_some() + { + return Err(WAITING_FOR_CONSUMER_REMOVAL.into()); + } + Ok(()) +} + +fn current_boundary(deployment: &Deployment, reg: &KarsSRERegistration) -> bool { + deployment.spec.as_ref().is_some_and(|spec| { + spec.template + .metadata + .as_ref() + .and_then(|m| m.annotations.as_ref()) + .is_some_and(|a| { + a.get(OWNER) == reg.metadata.uid.as_ref() && a.get(EPOCH) == Some(®.epoch()) + }) + }) +} + +pub(super) async fn validate_consumer( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let Some(deployment) = api + .get_opt("sre") + .await + .map_err(|e| api_error("Read SRE consumer", e))? + else { + return Ok(()); + }; + if current_boundary(&deployment, reg) { + return Ok(()); + } + if deployment + .spec + .as_ref() + .and_then(|s| s.template.metadata.as_ref()) + .and_then(|m| m.annotations.as_ref()) + .and_then(|a| a.get(OWNER)) + == reg.metadata.uid.as_ref() + { + return Ok(()); + } + let reviewed = reg + .spec + .legacy_consumer + .as_ref() + .ok_or("Existing SRE consumer requires an explicit UID/resourceVersion review")?; + let was_stopped = deployment + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(STOPPED)) + == reg.metadata.uid.as_ref(); + if deployment.metadata.uid.as_deref() != Some(reviewed.uid.as_str()) + || deployment.metadata.deletion_timestamp.is_some() + || (!was_stopped + && deployment.metadata.resource_version.as_deref() + != Some(reviewed.resource_version.as_str())) + { + return Err("Reviewed SRE consumer was changed or replaced".into()); + } + Ok(()) +} + +pub(super) async fn stop_legacy_consumer( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(deployment) = api + .get_opt("sre") + .await + .map_err(|e| api_error("Read SRE consumer for retirement", e))? + { + if reg.spec.enabled && current_boundary(&deployment, reg) { + return Ok(()); + } + validate_consumer(client, reg).await?; + if deployment.spec.as_ref().and_then(|spec| spec.replicas) != Some(0) { + api.patch("sre",&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":deployment.metadata.uid,"resourceVersion":deployment.metadata.resource_version, + "annotations":{STOPPED:reg.metadata.uid}}, + "spec":{"replicas":0}, + }))).await.map_err(|e|api_error("Stop reviewed SRE consumer",e))?; + } + } + let pods: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let pods = pods + .list(&ListParams::default()) + .await + .map_err(|e| api_error("Wait for old SRE consumers", e))?; + if pods.iter().any(|pod| { + pod.spec + .as_ref() + .and_then(|spec| spec.service_account_name.as_deref()) + == Some("sandbox") + }) { + return Err(WAITING_FOR_CONSUMERS.into()); + } + Ok(()) +} + +fn controller_managed(deployment: &Deployment, sandbox: &str) -> bool { + deployment + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get("kars.azure.com/sandbox")) + .map(String::as_str) + == Some(sandbox) + && deployment + .metadata + .managed_fields + .as_ref() + .is_some_and(|fields| { + fields.iter().any(|field| { + field.manager.as_deref() == Some(crate::field_managers::CLAWSANDBOX) + && field + .fields_v1 + .as_ref() + .is_some_and(|fields| fields.0.get("f:spec").is_some()) + }) + }) +} + +/// Existing #550 control credentials may have been exposed by the old SRE +/// identity. Rotate only proven controller-owned instances and restart their +/// owned consumer; the old router caches the token at startup. +pub(super) async fn rotate_owned_control_credentials( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let all: Api = Api::all(client.clone()); + let list = all + .list(&ListParams::default().fields("metadata.name=router-services-admin")) + .await + .map_err(|e| api_error("Inventory owned control credentials", e))?; + for secret in list { + if secret + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get("app.kubernetes.io/managed-by")) + .map(String::as_str) + != Some("kars-controller") + { + continue; + } + if secret.metadata.uid.as_deref().is_none_or(str::is_empty) + || secret + .metadata + .resource_version + .as_deref() + .is_none_or(str::is_empty) + || secret.metadata.deletion_timestamp.is_some() + || secret + .metadata + .owner_references + .as_ref() + .is_some_and(|owners| !owners.is_empty()) + { + return Err("Owned control credential identity is incomplete or terminating".into()); + } + let Some(namespace_name) = secret.namespace() else { + continue; + }; + let annotations = secret.metadata.annotations.as_ref(); + let Some(source_uid) = annotations.and_then(|a| a.get("kars.azure.com/sandbox-uid")) else { + continue; + }; + let Some(namespace_uid) = annotations.and_then(|a| a.get("kars.azure.com/namespace-uid")) + else { + continue; + }; + let namespaces: Api = Api::all(client.clone()); + let namespace = namespaces + .get(&namespace_name) + .await + .map_err(|e| api_error("Read control credential namespace", e))?; + if namespace.metadata.uid.as_deref() != Some(namespace_uid) + || namespace.metadata.deletion_timestamp.is_some() + { + return Err("Control credential namespace changed".into()); + } + let ns_annotations = namespace + .metadata + .annotations + .as_ref() + .ok_or("Control namespace claim is absent")?; + let workspace = ns_annotations + .get("kars.azure.com/sandbox-namespace") + .ok_or("Control namespace source is absent")?; + let name = ns_annotations + .get("kars.azure.com/sandbox-name") + .ok_or("Control namespace owner is absent")?; + let sandboxes: Api = Api::namespaced(client.clone(), workspace); + let sandbox = sandboxes + .get(name) + .await + .map_err(|e| api_error("Read control credential owner", e))?; + if sandbox.metadata.uid.as_deref() != Some(source_uid) + || sandbox.metadata.deletion_timestamp.is_some() + || !crate::reconciler::namespace_ownership::claimed(&namespace, &sandbox) + .map_err(|_| "Control namespace ownership is invalid")? + { + return Err("Control credential ownership changed; no rotation was authorized".into()); + } + check_secret_denial(client, &namespace_name).await?; + let deployments: Api = Api::namespaced(client.clone(), &namespace_name); + let Some(deployment) = deployments + .get_opt(name) + .await + .map_err(|e| api_error("Read control credential consumer", e))? + else { + return Err("Owned control credential has no verified consumer Deployment".into()); + }; + if !controller_managed(&deployment, name) { + return Err("Control credential consumer is not controller-owned".into()); + } + let epoch = reg.epoch(); + if annotations.and_then(|a| a.get(EPOCH)) != Some(&epoch) { + let secrets: Api = Api::namespaced(client.clone(), &namespace_name); + secrets.patch("router-services-admin",&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":secret.metadata.uid,"resourceVersion":secret.metadata.resource_version, + "annotations":{EPOCH:epoch}}, + "stringData":{"control-token":crate::providers::signing::generate_service_token()}, + }))).await.map_err(|e|api_error("Rotate owned control credential",e))?; + } + if deployment + .spec + .as_ref() + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|meta| meta.annotations.as_ref()) + .and_then(|a| a.get(EPOCH)) + != Some(&epoch) + { + deployments.patch(name,&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":deployment.metadata.uid,"resourceVersion":deployment.metadata.resource_version}, + "spec":{"template":{"metadata":{"annotations":{EPOCH:epoch}}}}, + }))).await.map_err(|e|api_error("Restart owned control credential consumer",e))?; + return Err(WAITING_FOR_ROTATION.into()); + } + let desired = deployment + .spec + .as_ref() + .and_then(|spec| spec.replicas) + .unwrap_or(1); + let ready = deployment.status.as_ref().is_some_and(|status| { + status.observed_generation == deployment.metadata.generation + && status.updated_replicas.unwrap_or(0) == desired + && status.available_replicas.unwrap_or(0) == desired + }); + if !ready { + return Err(WAITING_FOR_ROLLOUT.into()); + } + let selector = deployment + .spec + .as_ref() + .and_then(|spec| spec.selector.match_labels.as_ref()) + .filter(|labels| !labels.is_empty()) + .ok_or("Owned control credential consumer has no bounded Pod selector")? + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join(","); + let pods = Api::::namespaced(client.clone(), &namespace_name) + .list(&ListParams::default().labels(&selector)) + .await + .map_err(|e| api_error("Verify old control credential consumers have terminated", e))?; + if pods.iter().any(|pod| { + pod.metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(EPOCH)) + != Some(&epoch) + }) { + // Rollout availability alone can exclude a still-terminating old + // router which continues accepting its startup-cached control token. + return Err(WAITING_FOR_ROLLOUT.into()); + } + } + Ok(()) +} diff --git a/controller/src/sre_authority/pod.rs b/controller/src/sre_authority/pod.rs new file mode 100644 index 000000000..0067508b9 --- /dev/null +++ b/controller/src/sre_authority/pod.rs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + crd::KarsSandbox, + sre_registration::{AGENT_SECRET, EPOCH, KarsSRERegistration, NAME, OWNER, PRIVATE_SECRET}, +}; +use k8s_openapi::api::core::v1::{Namespace, Secret}; +use kube::{Api, Client, ResourceExt}; +use serde_json::{Value, json}; + +pub(crate) struct Projection { + pub epoch: String, + pub registration_uid: String, + pub tls_expiry: String, + pub credential_uid: String, +} + +pub(crate) async fn authorize( + client: &Client, + sandbox: &KarsSandbox, + namespace: &Namespace, +) -> Result, String> { + let reserved = sandbox.name_any() == "sre" + || sandbox + .metadata + .labels + .as_ref() + .and_then(|l| l.get("kars.azure.com/role")) + .map(String::as_str) + == Some("sre"); + if !reserved { + return Ok(None); + } + let registrations: Api = Api::all(client.clone()); + let reg = registrations + .get_opt(NAME) + .await + .map_err(|e| super::api_error("Read SRE enrollment", e))? + .ok_or("SRE source is staged but not enrolled by a cluster registrar")?; + if reg.spec.sandbox.uid != sandbox.uid().unwrap_or_default() + || reg.spec.sandbox.namespace != sandbox.namespace().unwrap_or_default() + || reg.spec.runtime_namespace.uid != namespace.uid().unwrap_or_default() + { + return Err("This SRE occupant is not the registered canonical source; no private identity is granted".into()); + } + let epoch = super::privacy_epoch(client, ®.spec.runtime_namespace.name) + .await? + .ok_or("SRE privacy epoch is absent")?; + if !reg.spec.enabled + || reg.status.as_ref().is_none_or(|status| { + status.phase != "Ready" + || status.observed_generation != reg.metadata.generation.unwrap_or_default() + || status.privacy_epoch.as_deref() != Some(epoch.as_str()) + || !status.legacy_secret_access_denied + }) + { + return Err("SRE authority migration has not reached Ready".into()); + } + let secrets: Api = Api::namespaced(client.clone(), ®.spec.runtime_namespace.name); + let secret = secrets + .get_metadata(PRIVATE_SECRET) + .await + .map_err(|e| super::api_error("Read SRE TLS revision", e))?; + let annotations = secret + .metadata + .annotations + .as_ref() + .ok_or("SRE private identity annotations missing")?; + if annotations.get(OWNER) != reg.metadata.uid.as_ref() || annotations.get(EPOCH) != Some(&epoch) + { + return Err("SRE private identity does not match the registered privacy epoch".into()); + } + Ok(Some(Projection { + epoch, + registration_uid: reg.uid().ok_or("SRE registration UID missing")?, + credential_uid: secret.uid().ok_or("SRE private credential UID missing")?, + tls_expiry: annotations + .get("kars.azure.com/sre-tls-expiry") + .cloned() + .ok_or("SRE TLS expiry missing")?, + })) +} + +fn env(container: &mut Value, name: &str, value: &str) { + let env = container["env"] + .as_array_mut() + .expect("controller container environment"); + env.retain(|entry| entry["name"] != name); + env.push(json!({"name":name,"value":value})); +} + +pub(crate) fn project(pod: &mut Value) { + // Do not change serviceAccountName: Azure federation remains + // system:serviceaccount::sandbox. Only the K8s diagnostic client + // uses the separate private identity. + pod["automountServiceAccountToken"] = false.into(); + let volumes = pod["volumes"] + .as_array_mut() + .expect("controller pod volumes"); + volumes.extend([ + json!({"name":"sre-api-private","secret":{"secretName":PRIVATE_SECRET}}), + json!({"name":"sre-api-agent","secret":{"secretName":AGENT_SECRET, + "items":[{"key":"token","path":"token"},{"key":"ca.crt","path":"ca.crt"},{"key":"namespace","path":"namespace"}]}}), + json!({"name":"router-kubernetes","projected":{"sources":[ + {"serviceAccountToken":{"path":"token","expirationSeconds":3600}}, + {"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}}, + {"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"fieldPath":"metadata.namespace"}}]}} + ]}}), + ]); + for container in pod["containers"] + .as_array_mut() + .expect("controller pod containers") + { + let router = container["name"] == "inference-router"; + container["volumeMounts"] + .as_array_mut() + .expect("controller mounts") + .push(json!({ + "name":if router {"router-kubernetes"} else {"sre-api-agent"}, + "mountPath":"/var/run/secrets/kubernetes.io/serviceaccount","readOnly":true, + })); + if router { + container["volumeMounts"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"sre-api-private","mountPath":"/etc/kars/sre-api","readOnly":true, + })); + env(container, "KARS_SRE_API_ENABLED", "true"); + container["readinessProbe"] = json!({ + "exec":{"command":["kars-inference-router","sre-ready"]}, + "initialDelaySeconds":3,"periodSeconds":5,"timeoutSeconds":5, + }); + } else { + let prior = container["env"] + .as_array() + .and_then(|values| values.iter().find(|entry| entry["name"] == "NO_PROXY")) + .and_then(|entry| entry["value"].as_str()) + .unwrap_or_default(); + let no_proxy = format!("127.0.0.1,localhost,{prior}"); + env(container, "KUBERNETES_SERVICE_HOST", "127.0.0.1"); + env(container, "KUBERNETES_SERVICE_PORT", "9446"); + env(container, "KUBERNETES_SERVICE_PORT_HTTPS", "9446"); + env(container, "NO_PROXY", &no_proxy); + env(container, "no_proxy", &no_proxy); + } + } +} + +pub(crate) fn annotations(projection: &Projection, agent_name: &str) -> Value { + json!({ + OWNER:projection.registration_uid,EPOCH:projection.epoch, + "kars.azure.com/sre-tls-expiry":projection.tls_expiry, + "kars.azure.com/sre-private-credential-uid":projection.credential_uid, + "azure.workload.identity/skip-containers":format!("{agent_name},egress-guard"), + }) +} diff --git a/controller/src/sre_authority/privacy_tests.rs b/controller/src/sre_authority/privacy_tests.rs new file mode 100644 index 000000000..c6b727112 --- /dev/null +++ b/controller/src/sre_authority/privacy_tests.rs @@ -0,0 +1,424 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::tests::{State, fixture, registration}; +use super::*; +use crate::sre_registration::*; +use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; + +const REG: &str = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical"; +const SECRETS: &str = "/api/v1/namespaces/kars-sre/secrets"; +const SA: &str = "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router"; +const CRBS: &str = "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings"; + +fn ready() -> KarsSRERegistration { + let mut reg = registration(); + reg.status = Some( + serde_json::from_value(json!({ + "phase":"Ready","observedGeneration":1,"privacyEpoch":reg.epoch(), + "routerServiceAccountUid":"router-sa","legacySecretAccessDenied":true, + "privacyRevision":crate::sre_privacy::REVISION, + })) + .unwrap(), + ); + reg +} + +pub(super) fn admission_ready(state: &Arc>) { + let mut state = state.lock().unwrap(); + for name in admission::POLICIES { + state.objects.insert(format!("/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/{name}"),json!({ + "metadata":{"name":name,"generation":1},"spec":{"failurePolicy":"Fail","validations":[]}, + "status":{"observedGeneration":1,"typeChecking":{}}})); + state.objects.insert( + format!( + "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings/{name}" + ), + json!({ + "metadata":{"name":name},"spec":{"policyName":name,"validationActions":["Deny"]}}), + ); + } +} + +fn alias(name: &str, account_name: &str, account_uid: Option<&str>) -> Value { + let mut secret = json!({"apiVersion":"v1","kind":"Secret","type":"kubernetes.io/service-account-token", + "metadata":{"name":name,"namespace":"kars-sre","uid":format!("uid-{name}"),"resourceVersion":"1", + "annotations":{"kubernetes.io/service-account.name":account_name}}, + "data":{"token":"PRIVATE_TOKEN_SENTINEL"}}); + if let Some(uid) = account_uid { + secret["metadata"]["annotations"]["kubernetes.io/service-account.uid"] = uid.into(); + } + secret +} + +#[tokio::test] +async fn prestaged_aliases_and_recreated_uids_never_receive_a_service_account_or_credentials() { + for (name, uid) in [ + ("sre-api-router", None), + ("sre-api-router", Some("obsolete-sa")), + ("renamed-before-policy", Some("router-sa")), + ] { + let (_server, client, state) = fixture().await; + let reg = ready(); + state.lock().unwrap().objects.insert( + format!("{SECRETS}/arbitrary-alias"), + alias("arbitrary-alias", name, uid), + ); + let authority = live::verify(&client, ®).await.unwrap(); + let error = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap_err(); + assert!(!error.contains("PRIVATE_TOKEN_SENTINEL")); + let state = state.lock().unwrap(); + assert!(state.calls.iter().all(|(method, _, _)| method == "GET")); + assert!( + state + .objects + .contains_key(&format!("{SECRETS}/arbitrary-alias")) + ); + assert!( + state + .metadata_requests + .iter() + .all(|accept| accept.contains("PartialObjectMetadataList")) + ); + } +} + +#[tokio::test] +async fn unsafe_alias_after_ready_revokes_owned_grants_but_preserves_foreign_and_unrelated_subjects() + { + let (_server, client, state) = fixture().await; + let reg = ready(); + admission_ready(&state); + { + let mut state = state.lock().unwrap(); + state + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + state.objects.insert( + format!("{SECRETS}/alias"), + alias("alias", "sre-api-router", Some("router-sa")), + ); + state.objects.insert( + SA.into(), + json!({"metadata":{"name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE, + "uid":"router-sa","resourceVersion":"1"}}), + ); + for (name, role, owner) in [ + ( + "kars-sre-private-reader", + "kars-sre-private-diagnostics", + "registration", + ), + ( + "kars-sre-private-author", + "kars-sre-action-author", + "foreign", + ), + ( + "kars-sre-private-renew", + "kars-sre-router-renew", + "registration", + ), + ] { + let mut binding = json!({"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":name,"uid":format!("uid-{name}"),"resourceVersion":"1", + "annotations":{OWNER:owner,"kars.azure.com/namespace-uid":"runtime-ns"}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":role}, + "subjects":[{"kind":"ServiceAccount","name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE}]}); + if name == "kars-sre-private-renew" { + binding["subjects"].as_array_mut().unwrap().push(json!({"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"})); + } + state.objects.insert(format!("{CRBS}/{name}"), binding); + } + } + assert!(reconcile(&client, ®).await.is_err()); + let state = state.lock().unwrap(); + assert!(state.objects.contains_key(&format!("{SECRETS}/alias"))); + assert!( + !state + .objects + .contains_key(&format!("{CRBS}/kars-sre-private-reader")) + ); + assert!( + state + .objects + .contains_key(&format!("{CRBS}/kars-sre-private-author")) + ); + assert_eq!( + state.objects[&format!("{CRBS}/kars-sre-private-renew")]["subjects"], + json!([{"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]) + ); + assert_eq!(state.objects[REG]["status"]["phase"], "Blocked"); + assert_eq!( + state.objects[REG]["status"]["legacySecretAccessDenied"], + false + ); + let patch = &state + .calls + .iter() + .find(|(_, path, _)| path.ends_with("/status")) + .unwrap() + .2; + assert!(patch["status"]["privacyEpoch"].is_null()); + assert!(patch["status"]["privacyRevision"].is_null()); + assert!(state.calls.iter().all(|(method, path, _)| method == "GET" + || path.starts_with(CRBS) + || path.ends_with("/status"))); +} + +#[tokio::test] +async fn watch_only_and_wildcard_group_grants_fail_review_before_mutation() { + for rule in [ + json!({"apiGroups":[""],"resources":["secrets"],"verbs":["watch"]}), + json!({"apiGroups":["*"],"resources":["*"],"verbs":["watch"]}), + json!({"apiGroups":[""],"resources":["secrets"],"verbs":["*"]}), + json!({"apiGroups":[""],"resources":["pods","secrets"],"verbs":["get","watch"]}), + ] { + let (_server, client, state) = fixture().await; + { + let mut state = state.lock().unwrap(); + state.binding["subjects"] = json!([{"kind":"Group","name":"system:serviceaccounts:kars-sre", + "apiGroup":"rbac.authorization.k8s.io"}]); + state.objects.insert( + "/apis/rbac.authorization.k8s.io/v1/clusterroles/kars-sre-reader".into(), + json!({"metadata":{"name":"kars-sre-reader"},"rules":[rule]}), + ); + } + assert!( + bindings::review(&client, ®istration()) + .await + .err() + .unwrap() + .contains("broad group") + ); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); + } +} + +#[tokio::test] +async fn live_reviews_cover_namespace_and_cluster_watch_including_named_grants() { + for (namespace, name) in [ + (Some("kars-sre"), None), + (None, None), + (Some("kars-sre"), Some(PRIVATE_SECRET)), + (None, Some("router-services-admin")), + ] { + let (_server, client, state) = fixture().await; + state.lock().unwrap().watch_allowed = + Some((namespace.map(str::to_owned), name.map(str::to_owned))); + assert!(check_secret_denial(&client, "kars-sre").await.is_err()); + assert!(privacy_epoch(&client, "kars-sre").await.is_err()); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .any(|(_, _, request)| request["spec"]["resourceAttributes"]["verb"] == "watch") + ); + } +} + +#[tokio::test] +async fn inventory_api_errors_and_prior_get_list_only_status_are_not_privacy_safe() { + let (_server, client, state) = fixture().await; + admission_ready(&state); + let mut reg = ready(); + reg.status.as_mut().unwrap().privacy_revision = None; + state + .lock() + .unwrap() + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + assert!(privacy_epoch(&client, RUNTIME_NAMESPACE).await.is_err()); + state.lock().unwrap().api_error = Some(403); + let error = credential_guard::scan(&client, ®).await.unwrap_err(); + assert!(!error.contains("PRIVATE_SENTINEL")); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, path, _)| method == "GET" || path.ends_with("/subjectaccessreviews")) + ); +} + +#[tokio::test] +async fn blocked_recovery_replaces_the_owned_token_anchor_uid_without_recreating_pod_identity() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let path = format!("{SECRETS}/{PRIVATE_SECRET}"); + let old_uid = state.lock().unwrap().objects[&path]["metadata"]["uid"].clone(); + reg.status = ready().status; + reg.status.as_mut().unwrap().phase = "Blocked".into(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let state = state.lock().unwrap(); + assert_ne!(state.objects[&path]["metadata"]["uid"], old_uid); + assert_eq!( + state + .calls + .iter() + .filter(|(method, path, _)| method == "DELETE" && path.ends_with(PRIVATE_SECRET)) + .count(), + 1 + ); + assert!( + !state + .calls + .iter() + .any(|(method, path, _)| method == "DELETE" && path.contains("/serviceaccounts/")) + ); +} + +#[tokio::test] +async fn newly_discovered_watch_group_revokes_previously_ready_owned_reader() { + let (_server, client, state) = fixture().await; + let reg = ready(); + admission_ready(&state); + { + let mut state = state.lock().unwrap(); + state + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + state.binding["subjects"] = json!([{"kind":"Group","name":"system:serviceaccounts:kars-sre", + "apiGroup":"rbac.authorization.k8s.io"}]); + state.objects.insert("/apis/rbac.authorization.k8s.io/v1/clusterroles/kars-sre-reader".into(), + json!({"metadata":{"name":"kars-sre-reader"},"rules":[{"apiGroups":[""],"resources":["secrets"],"verbs":["watch"]}]})); + state.objects.insert(format!("{CRBS}/kars-sre-private-reader"),json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":"kars-sre-private-reader","uid":"owned-reader","resourceVersion":"1", + "annotations":{OWNER:"registration","kars.azure.com/namespace-uid":"runtime-ns"}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"kars-sre-private-diagnostics"}, + "subjects":[{"kind":"ServiceAccount","name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE}]})); + } + assert!(reconcile(&client, ®).await.is_err()); + let state = state.lock().unwrap(); + assert!( + !state + .objects + .contains_key(&format!("{CRBS}/kars-sre-private-reader")) + ); + assert_eq!(state.objects[REG]["status"]["phase"], "Blocked"); + assert!(state.calls.iter().all(|(method, path, _)| method == "GET" + || path == &format!("{CRBS}/kars-sre-private-reader") + || path.ends_with("/status"))); +} + +#[tokio::test] +async fn annotation_changes_and_secret_recreation_are_rescanned_and_preserved() { + let (_server, client, state) = fixture().await; + let reg = ready(); + let path = format!("{SECRETS}/alias"); + state + .lock() + .unwrap() + .objects + .insert(path.clone(), alias("alias", "ordinary", None)); + credential_guard::scan(&client, ®).await.unwrap(); + for uid in ["original", "recreated"] { + { + let mut state = state.lock().unwrap(); + state.objects.get_mut(&path).unwrap()["metadata"]["uid"] = uid.into(); + state.objects.get_mut(&path).unwrap()["metadata"]["annotations"]["kubernetes.io/service-account.name"] = + ROUTER_SA.into(); + } + assert!(credential_guard::scan(&client, ®).await.is_err()); + assert_eq!(state.lock().unwrap().objects[&path]["metadata"]["uid"], uid); + } + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn quarantine_retires_owned_identity_but_preserves_the_unsafe_unknown_alias() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + reg.status = ready().status; + reg.status.as_mut().unwrap().router_service_account_uid = sa.metadata.uid.clone(); + admission_ready(&state); + { + let mut state = state.lock().unwrap(); + state + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + state.objects.insert( + format!("{SECRETS}/unsafe-alias"), + alias("unsafe-alias", ROUTER_SA, sa.metadata.uid.as_deref()), + ); + state.calls.clear(); + } + assert!(reconcile(&client, ®).await.is_err()); + let state = state.lock().unwrap(); + assert!( + state + .objects + .contains_key(&format!("{SECRETS}/unsafe-alias")) + ); + assert!(!state.objects.contains_key(SA)); + assert!( + !state + .objects + .contains_key(&format!("{SECRETS}/{PRIVATE_SECRET}")) + ); + assert!(state.calls.iter().all(|(method, path, _)| { + method != "DELETE" + || [ + SA, + &format!("{SECRETS}/{PRIVATE_SECRET}"), + &format!("{SECRETS}/{AGENT_SECRET}"), + ] + .contains(&path.as_str()) + })); +} + +#[tokio::test] +async fn private_service_account_replacement_uid_is_never_deleted_during_quarantine() { + let (_server, client, state) = fixture().await; + let reg = ready(); + state.lock().unwrap().objects.insert(SA.into(),json!({ + "metadata":{"name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE,"uid":"replacement","resourceVersion":"2", + "annotations":{OWNER:"registration","kars.azure.com/sandbox-uid":"source","kars.azure.com/namespace-uid":"runtime-ns"}}})); + assert!(credentials::retire(&client, ®).await.is_err()); + assert!(state.lock().unwrap().objects.contains_key(SA)); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} diff --git a/controller/src/sre_authority/retirement_tests.rs b/controller/src/sre_authority/retirement_tests.rs new file mode 100644 index 000000000..cdc709f21 --- /dev/null +++ b/controller/src/sre_authority/retirement_tests.rs @@ -0,0 +1,561 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::privacy_tests::admission_ready; +use super::tests::{State, fixture, registration}; +use super::{bindings, migration, reconcile}; +use crate::sre_registration::{ + BindingReview, ConsumerReview, EPOCH, KarsSRERegistration, OWNER, RUNTIME_NAMESPACE, + RegistrationStatus, +}; +use k8s_openapi::api::rbac::v1::{ClusterRoleBinding, RoleBinding}; +use kube::{ + Api, + api::{Patch, PatchParams}, +}; +use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; + +const REG: &str = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical"; +const CRBS: &str = "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings"; +const CONSUMER: &str = "/apis/apps/v1/namespaces/kars-sre/deployments/sre"; +const RETIRED: &str = "kars.azure.com/sre-legacy-retired"; + +fn stopped_owned_consumer(reg: &KarsSRERegistration) -> Value { + json!({"apiVersion":"apps/v1","kind":"Deployment", + "metadata":{"name":"sre","namespace":RUNTIME_NAMESPACE,"uid":"owned-consumer","resourceVersion":"5"}, + "spec":{"replicas":0,"selector":{"matchLabels":{"app":"sre"}}, + "template":{"metadata":{"annotations":{OWNER:reg.metadata.uid,EPOCH:reg.epoch()}}, + "spec":{"containers":[],"serviceAccountName":"sandbox"}}}}) +} + +fn seed_stopped_consumer(state: &Arc>, reg: &KarsSRERegistration) { + let mut locked = state.lock().unwrap(); + locked + .objects + .insert(CONSUMER.into(), stopped_owned_consumer(reg)); + locked.objects.insert( + "/api/v1/namespaces/kars-sre/pods".into(), + json!({"apiVersion":"v1","kind":"PodList","metadata":{},"items":[]}), + ); +} + +#[tokio::test] +async fn disabled_retirement_removes_only_quiesced_owned_deployment_with_uid_rv_fences() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap(); + migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap(); + let locked = state.lock().unwrap(); + assert!(!locked.objects.contains_key(CONSUMER)); + let deletes: Vec<_> = locked + .calls + .iter() + .filter(|(method, _, _)| method == "DELETE") + .collect(); + assert_eq!(deletes.len(), 1); + assert_eq!(deletes[0].1, CONSUMER); + assert_eq!( + deletes[0].2["preconditions"], + json!({"uid":"owned-consumer","resourceVersion":"5"}) + ); +} + +#[tokio::test] +async fn retired_audit_record_repairs_its_leftover_owned_consumer_without_reissuing_authority() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + reg.status = Some(RegistrationStatus { + phase: "Retired".into(), + observed_generation: reg.metadata.generation.unwrap_or_default(), + privacy_revision: Some(crate::sre_privacy::REVISION.into()), + ..Default::default() + }); + seed_stopped_consumer(&state, ®); + reconcile(&client, ®).await.unwrap(); + let locked = state.lock().unwrap(); + assert!(!locked.objects.contains_key(CONSUMER)); + assert!(locked.calls.iter().all(|(method, path, _)| method == "GET" + || method == "DELETE" + || method == "POST" && path.ends_with("/subjectaccessreviews"))); +} + +#[tokio::test] +async fn retirement_preserves_foreign_namespace_or_unowned_consumer() { + for foreign_namespace in [true, false] { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + { + let mut locked = state.lock().unwrap(); + if foreign_namespace { + locked.namespace["metadata"]["uid"] = "foreign-namespace".into(); + } else { + locked.objects.get_mut(CONSUMER).unwrap()["spec"]["template"]["metadata"]["annotations"] + [OWNER] = "foreign-registration".into(); + } + } + let before = state.lock().unwrap().objects[CONSUMER].clone(); + migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap(); + let locked = state.lock().unwrap(); + assert_eq!(locked.objects[CONSUMER], before); + assert!(locked.calls.iter().all(|(method, _, _)| method == "GET")); + } +} + +#[tokio::test] +async fn retirement_rechecks_uid_ownership_and_quiescence_before_deleting() { + use std::sync::atomic::{AtomicUsize, Ordering}; + for field in ["uid", "owner", "replicas"] { + let (server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + let value = stopped_owned_consumer(®); + let reads = AtomicUsize::new(0); + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path(CONSUMER)) + .respond_with(move |_: &wiremock::Request| { + let mut current = value.clone(); + if reads.fetch_add(1, Ordering::SeqCst) >= 3 { + match field { + "uid" => current["metadata"]["uid"] = "replacement".into(), + "owner" => { + current["spec"]["template"]["metadata"]["annotations"][OWNER] = + "foreign".into() + } + _ => current["spec"]["replicas"] = 1.into(), + } + } + wiremock::ResponseTemplate::new(200).set_body_json(current) + }) + .with_priority(1) + .mount(&server) + .await; + let error = migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap_err(); + assert!(error.contains("changed after quiescence")); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method != "DELETE") + ); + } +} + +#[tokio::test] +async fn retirement_delete_conflict_is_not_retried_or_forced() { + for code in [403, 409] { + let (server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + wiremock::Mock::given(wiremock::matchers::method("DELETE")) + .and(wiremock::matchers::path(CONSUMER)) + .respond_with(wiremock::ResponseTemplate::new(code).set_body_json(json!({ + "apiVersion":"v1","kind":"Status","status":"Failure", + "reason":if code == 409 {"Conflict"} else {"Forbidden"},"code":code, + "message":"PRIVATE_SENTINEL"}))) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + let error = migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap_err(); + assert_eq!( + error, + format!("Remove stopped owned SRE consumer: Kubernetes status {code}") + ); + assert!(!error.contains("PRIVATE_SENTINEL")); + assert!(state.lock().unwrap().objects.contains_key(CONSUMER)); + } +} + +#[tokio::test] +async fn retirement_waits_for_deletion_without_removing_foreign_finalizers() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + { + let mut locked = state.lock().unwrap(); + let deployment = locked.objects.get_mut(CONSUMER).unwrap(); + deployment["metadata"]["deletionTimestamp"] = "2026-09-09T00:00:00Z".into(); + deployment["metadata"]["finalizers"] = json!(["e2e.example/foreign"]); + } + let error = migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap_err(); + assert!(migration::is_waiting(&error)); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); + assert_eq!( + state.lock().unwrap().objects[CONSUMER]["metadata"]["finalizers"], + json!(["e2e.example/foreign"]) + ); +} + +fn reviews(state: &Arc>, kind: &str) -> (KarsSRERegistration, String) { + admission_ready(state); + let mut locked = state.lock().unwrap(); + locked.binding["metadata"]["annotations"] = json!({"e2e-retained":"yes"}); + let local = kind == "RoleBinding"; + let path = if local { + "/apis/rbac.authorization.k8s.io/v1/namespaces/kars-sre/rolebindings/second".into() + } else { + format!("{CRBS}/second") + }; + let mut second = json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":kind, + "metadata":{"name":"second","uid":"second-binding","resourceVersion":"7", + "annotations":{"e2e-retained":"yes"}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":if local {"Role"} else {"ClusterRole"}, + "name":"custom-retirement-role"},"subjects":locked.binding["subjects"], + }); + if local { + second["metadata"]["namespace"] = RUNTIME_NAMESPACE.into(); + } + let mut reg = registration(); + reg.spec.legacy_bindings.push(BindingReview { + kind: kind.into(), + namespace: local.then(|| RUNTIME_NAMESPACE.into()), + name: "second".into(), + uid: "second-binding".into(), + resource_version: "7".into(), + role_ref: serde_json::from_value(second["roleRef"].clone()).unwrap(), + subjects: serde_json::from_value(second["subjects"].clone()).unwrap(), + }); + reg.spec.legacy_consumer = Some(ConsumerReview { + namespace: RUNTIME_NAMESPACE.into(), + name: "sre".into(), + uid: "consumer".into(), + resource_version: "1".into(), + }); + locked.objects.insert(path.clone(), second); + locked.objects.insert( + CONSUMER.into(), + json!({"apiVersion":"apps/v1","kind":"Deployment", + "metadata":{"name":"sre","namespace":"kars-sre","uid":"consumer","resourceVersion":"1"}, + "spec":{"replicas":1,"selector":{"matchLabels":{"app":"sre"}}, + "template":{"metadata":{},"spec":{"containers":[],"serviceAccountName":"sandbox"}}}}), + ); + locked + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + for (name, rule) in [ + ( + "kars-sre-private-diagnostics", + json!({"apiGroups":[""],"resources":["pods"],"verbs":["get"]}), + ), + ( + "kars-sre-action-author", + json!({"apiGroups":["kars.azure.com"],"resources":["karssreactions"],"verbs":["create"]}), + ), + ( + "kars-sre-router-renew", + json!({"apiGroups":["kars.azure.com"],"resources":["karssreregistrations"], + "resourceNames":["canonical"],"verbs":["get","renew"]}), + ), + ( + "custom-retirement-role", + json!({"apiGroups":[""],"resources":["limitranges"],"verbs":["get"]}), + ), + ] { + let role_path = if local && name == "custom-retirement-role" { + format!("/apis/rbac.authorization.k8s.io/v1/namespaces/kars-sre/roles/{name}") + } else { + format!("/apis/rbac.authorization.k8s.io/v1/clusterroles/{name}") + }; + locked + .objects + .insert(role_path, json!({"metadata":{"name":name},"rules":[rule]})); + } + (reg, path) +} + +fn survivors() -> Value { + json!([{"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]) +} + +#[tokio::test] +async fn retirement_http_fixture_dry_runs_never_mutate_either_binding_store() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + let (_server, client, state) = fixture().await; + let (_reg, path) = reviews(&state, kind); + let (before, second) = { + let locked = state.lock().unwrap(); + (locked.binding.clone(), locked.objects[&path].clone()) + }; + let params = PatchParams { + dry_run: true, + ..Default::default() + }; + let patch = json!({"metadata":{"uid":"binding","resourceVersion":"1", + "annotations":{RETIRED:"registration"}},"subjects":survivors()}); + let projected = Api::::all(client.clone()) + .patch("legacy", ¶ms, &Patch::Merge(&patch)) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(projected).unwrap()["subjects"], + survivors() + ); + let patch = json!({"metadata":{"uid":"second-binding","resourceVersion":"7", + "annotations":{RETIRED:"registration"}},"subjects":survivors()}); + if kind == "RoleBinding" { + Api::::namespaced(client, RUNTIME_NAMESPACE) + .patch("second", ¶ms, &Patch::Merge(&patch)) + .await + .unwrap(); + } else { + Api::::all(client) + .patch("second", ¶ms, &Patch::Merge(&patch)) + .await + .unwrap(); + } + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before); + assert_eq!(locked.objects[&path], second); + assert_eq!(locked.binding_patches.len(), 2); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + } +} + +#[tokio::test] +async fn retirement_all_preflights_precede_identical_cas_writes_and_preserve_survivors() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, kind); + let before = state.lock().unwrap().objects.clone(); + let reviewed = bindings::review(&client, ®).await.unwrap(); + bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap(); + { + let locked = state.lock().unwrap(); + let patches = &locked.binding_patches; + assert_eq!(patches.len(), 4); + assert_eq!( + patches + .iter() + .map(|(_, dry_run, _)| *dry_run) + .collect::>(), + [true, true, false, false] + ); + for (preflight, write) in patches[..2].iter().zip(&patches[2..]) { + assert_eq!(preflight.0, write.0); + assert_eq!(preflight.2, write.2); + assert_eq!(preflight.2["subjects"], survivors()); + assert_eq!( + preflight.2["metadata"]["annotations"][RETIRED], + "registration" + ); + } + assert_eq!(patches[0].0, format!("{CRBS}/legacy")); + assert_eq!(patches[1].0, path); + assert_eq!(patches[0].2["metadata"]["uid"], "binding"); + assert_eq!(patches[0].2["metadata"]["resourceVersion"], "1"); + assert_eq!(patches[1].2["metadata"]["uid"], "second-binding"); + assert_eq!(patches[1].2["metadata"]["resourceVersion"], "7"); + for binding in [&locked.binding, &locked.objects[&path]] { + assert_eq!(binding["subjects"], survivors()); + assert_eq!(binding["metadata"]["annotations"]["e2e-retained"], "yes"); + assert_eq!(binding["metadata"]["annotations"][RETIRED], "registration"); + } + assert_eq!(locked.binding["metadata"]["resourceVersion"], "2"); + assert_eq!(locked.objects[&path]["metadata"]["resourceVersion"], "8"); + assert_eq!( + locked.binding["roleRef"], + serde_json::to_value(®.spec.legacy_bindings[0].role_ref).unwrap() + ); + assert_eq!(locked.objects[&path]["roleRef"], before[&path]["roleRef"]); + assert_eq!(locked.objects[CONSUMER], before[CONSUMER]); + assert!( + locked + .calls + .iter() + .all(|(method, _, _)| method == "GET" || method == "PATCH") + ); + } + let reviewed = bindings::review(&client, ®).await.unwrap(); + bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap(); + assert_eq!(state.lock().unwrap().binding_patches.len(), 4); + } +} + +#[tokio::test] +async fn retirement_second_binding_forbidden_preserves_all_grants_roles_and_consumer() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, kind); + let (before, mut before_objects) = { + let mut locked = state.lock().unwrap(); + locked + .binding_patch_errors + .insert((path.clone(), true), 403); + (locked.binding.clone(), locked.objects.clone()) + }; + let error = reconcile(&client, ®).await.unwrap_err(); + assert_eq!( + error, + format!("Preflight reviewed SRE {kind} retirement: Kubernetes status 403") + ); + assert!(!error.contains("PRIVATE_SENTINEL")); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before); + assert_eq!(locked.objects[REG]["status"]["phase"], "Blocked"); + let mut after_objects = locked.objects.clone(); + before_objects.remove(REG); + after_objects.remove(REG); + assert_eq!(after_objects, before_objects); + assert_eq!(locked.binding_patches.len(), 2); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + assert_eq!(locked.binding_patches[0].0, format!("{CRBS}/legacy")); + assert_eq!(locked.binding_patches[1].0, path); + assert!( + locked + .calls + .iter() + .all(|(method, _, _)| method == "GET" || method == "PATCH") + ); + assert!( + !locked + .calls + .iter() + .any(|(method, path, _)| method == "PATCH" && path == CONSUMER) + ); + } +} + +#[tokio::test] +async fn retirement_preflight_rejects_stale_uid_or_resource_version_before_any_write() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + for (field, changed, code) in [("uid", "replacement", 422), ("resourceVersion", "8", 409)] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, kind); + let reviewed = bindings::review(&client, ®).await.unwrap(); + let before = { + let mut locked = state.lock().unwrap(); + locked.objects.get_mut(&path).unwrap()["metadata"][field] = changed.into(); + (locked.binding.clone(), locked.objects.clone()) + }; + let error = bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap_err(); + assert_eq!( + error, + format!("Preflight reviewed SRE {kind} retirement: Kubernetes status {code}") + ); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before.0); + assert_eq!(locked.objects, before.1); + assert_eq!(locked.binding_patches.len(), 2); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + } + } +} + +#[tokio::test] +async fn retirement_preflight_propagates_other_api_failures_without_writes() { + for code in [409, 422, 500] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, "ClusterRoleBinding"); + let reviewed = bindings::review(&client, ®).await.unwrap(); + let before = { + let mut locked = state.lock().unwrap(); + locked.binding_patch_errors.insert((path, true), code); + (locked.binding.clone(), locked.objects.clone()) + }; + let error = bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap_err(); + assert_eq!( + error, + format!( + "Preflight reviewed SRE ClusterRoleBinding retirement: Kubernetes status {code}" + ) + ); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before.0); + assert_eq!(locked.objects, before.1); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + } +} + +#[tokio::test] +async fn retirement_write_phase_races_fail_closed_without_claiming_atomic_rollback() { + for code in [403, 409] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, "RoleBinding"); + let reviewed = bindings::review(&client, ®).await.unwrap(); + let before = { + let mut locked = state.lock().unwrap(); + locked + .binding_patch_errors + .insert((path.clone(), false), code); + locked.objects.clone() + }; + let error = bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap_err(); + assert_eq!( + error, + format!("Retire reviewed SRE RoleBinding: Kubernetes status {code}") + ); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding["subjects"], survivors()); + assert_eq!(locked.objects, before); + assert_eq!(locked.binding_patches.len(), 4); + assert_eq!( + locked + .binding_patches + .iter() + .map(|(_, dry_run, _)| *dry_run) + .collect::>(), + [true, true, false, false] + ); + } +} diff --git a/controller/src/sre_authority/tests.rs b/controller/src/sre_authority/tests.rs new file mode 100644 index 000000000..c4e015b2e --- /dev/null +++ b/controller/src/sre_authority/tests.rs @@ -0,0 +1,604 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use crate::sre_registration::*; +use base64::Engine; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +pub(super) fn registration() -> KarsSRERegistration { + serde_json::from_value(json!({ + "apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSRERegistration", + "metadata":{"name":"canonical","uid":"registration","resourceVersion":"1","generation":1}, + "spec":{"controller":{"namespace":{"name":"kars-system","uid":"control-ns"}, + "deployment":{"name":"kars-controller","uid":"controller"},"release":"kars"}, + "sandbox":{"namespace":"kars-system","name":"sre","uid":"source"}, + "runtimeNamespace":{"name":"kars-sre","uid":"runtime-ns"},"enabled":true, + "legacyBindings":[{"kind":"ClusterRoleBinding","name":"legacy","uid":"binding","resourceVersion":"1", + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"kars-sre-reader"}, + "subjects":[{"kind":"ServiceAccount","name":"sandbox","namespace":"kars-sre"}, + {"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]}]}, + })).unwrap() +} + +fn sandbox() -> Value { + json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSandbox", + "metadata":{"name":"sre","namespace":"kars-system","uid":"source","resourceVersion":"1", + "labels":{"kars.azure.com/role":"sre"},"annotations":{"kars.azure.com/namespace-uid":"runtime-ns"}}, + "spec":{"runtime":{"kind":"Hermes","hermes":{}},"inferenceRef":{"name":"sre-inference"}}}) +} + +fn namespace() -> Value { + json!({"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kars-sre","uid":"runtime-ns","resourceVersion":"1", + "annotations":{"kars.azure.com/namespace-claim-version":"v1","kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre","kars.azure.com/sandbox-uid":"source"}}}) +} + +fn binding() -> Value { + json!({"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":"legacy","uid":"binding","resourceVersion":"1"}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"kars-sre-reader"}, + "subjects":[{"kind":"ServiceAccount","name":"sandbox","namespace":"kars-sre"}, + {"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]}) +} + +fn api_error(code: u16) -> ResponseTemplate { + ResponseTemplate::new(code).set_body_json(json!({"apiVersion":"v1","kind":"Status","status":"Failure", + "code":code,"reason":match code {404=>"NotFound",409=>"Conflict",422=>"Invalid",_=>"Forbidden"}, + "message":"PRIVATE_SENTINEL"})) +} + +fn patched_binding(existing: &Value, patch: &Value, dry_run: bool) -> Value { + let mut value = existing.clone(); + value["subjects"] = patch["subjects"].clone(); + if let Some(annotations) = patch["metadata"]["annotations"].as_object() { + if !value["metadata"]["annotations"].is_object() { + value["metadata"]["annotations"] = json!({}); + } + for (key, annotation) in annotations { + value["metadata"]["annotations"][key] = annotation.clone(); + } + } + if !dry_run { + value["metadata"]["resourceVersion"] = (existing["metadata"]["resourceVersion"] + .as_str() + .unwrap() + .parse::() + .unwrap() + + 1) + .to_string() + .into(); + } + value +} + +pub(super) struct State { + pub(super) sandbox: Value, + pub(super) namespace: Value, + pub(super) binding: Value, + pub(super) allowed: bool, + pub(super) calls: Vec<(String, String, Value)>, + pub(super) api_error: Option, + pub(super) objects: BTreeMap, + pub(super) watch_allowed: Option<(Option, Option)>, + pub(super) metadata_requests: Vec, + pub(super) binding_patches: Vec<(String, bool, Value)>, + pub(super) binding_patch_errors: BTreeMap<(String, bool), u16>, +} + +pub(super) async fn fixture() -> (MockServer, Client, Arc>) { + let state = Arc::new(Mutex::new(State { + sandbox: sandbox(), + namespace: namespace(), + binding: binding(), + allowed: false, + calls: Vec::new(), + api_error: None, + objects: BTreeMap::new(), + watch_allowed: None, + metadata_requests: Vec::new(), + binding_patches: Vec::new(), + binding_patch_errors: BTreeMap::new(), + })); + let server = MockServer::start().await; + let handler = state.clone(); + Mock::given(|_:&wiremock::Request|true).respond_with(move |request:&wiremock::Request| { + let mut state=handler.lock().unwrap(); + let path=request.url.path(); + let body:Value=request.body_json().unwrap_or(Value::Null); + let dry_run=request.url.query_pairs().any(|(key,value)|key=="dryRun" && value=="All"); + state.calls.push((request.method.to_string(),path.into(),body.clone())); + if request.method == "PATCH" && body["subjects"].is_array() { + state.binding_patches.push((path.into(),dry_run,body.clone())); + if let Some(code)=state.binding_patch_errors.get(&(path.into(),dry_run)) { + return api_error(*code); + } + } + if request.method == "GET" && path=="/api/v1/namespaces/kars-sre/secrets" { + state.metadata_requests.push(request.headers.get("accept").unwrap().to_str().unwrap().into()); + } + if let Some(code)=state.api_error {return api_error(code)} + if request.method == "GET" && let Some(value) = state.objects.get(path) { + return ResponseTemplate::new(200).set_body_json(value); + } + if request.method == "DELETE" && let Some(value) = state.objects.get(path) { + if body["preconditions"]["uid"] != value["metadata"]["uid"] || + body["preconditions"]["resourceVersion"] != value["metadata"]["resourceVersion"] { return api_error(409); } + state.objects.remove(path); + return ResponseTemplate::new(200).set_body_json(json!({"kind":"Status","apiVersion":"v1","status":"Success"})); + } + if request.method == "POST" && (path.ends_with("/secrets") || path.ends_with("/serviceaccounts")) { + let mut value = body.clone(); + value["metadata"]["uid"] = format!("uid-{}-{}", body["metadata"]["name"].as_str().unwrap(),state.calls.len()).into(); + value["metadata"]["resourceVersion"] = "1".into(); + state.objects.insert(format!("{path}/{}",body["metadata"]["name"].as_str().unwrap()),value.clone()); + return ResponseTemplate::new(201).set_body_json(value); + } + if request.method == "PATCH" && path.contains("/secrets/") { + let Some(value) = state.objects.get_mut(path) else { return api_error(404); }; + if body["metadata"]["uid"] != value["metadata"]["uid"] || + body["metadata"]["resourceVersion"] != value["metadata"]["resourceVersion"] { return api_error(409); } + for (key, annotation) in body["metadata"]["annotations"].as_object().unwrap() { + value["metadata"]["annotations"][key] = annotation.clone(); + } + for (key, material) in body["stringData"].as_object().unwrap() { + value["data"][key] = base64::engine::general_purpose::STANDARD.encode(material.as_str().unwrap()).into(); + } + value["metadata"]["resourceVersion"] = + (value["metadata"]["resourceVersion"].as_str().unwrap().parse::().unwrap()+1).to_string().into(); + return ResponseTemplate::new(200).set_body_json(value.clone()); + } + if request.method == "PATCH" && body["subjects"].is_array() && state.objects.contains_key(path) { + let value=&state.objects[path]; + if body["metadata"]["uid"]!=value["metadata"]["uid"] {return api_error(422);} + if body["metadata"]["resourceVersion"]!=value["metadata"]["resourceVersion"] {return api_error(409);} + let value=patched_binding(value,&body,dry_run); + if !dry_run {state.objects.insert(path.into(),value.clone());} + return ResponseTemplate::new(200).set_body_json(value); + } + let value=match (request.method.as_str(),path) { + ("GET","/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical")=>serde_json::to_value(registration()).unwrap(), + ("GET","/api/v1/namespaces/kars-system")=>json!({"metadata":{"name":"kars-system","uid":"control-ns","resourceVersion":"1"}}), + ("GET","/apis/apps/v1/namespaces/kars-system/deployments/kars-controller")=>json!({ + "metadata":{"name":"kars-controller","namespace":"kars-system","uid":"controller","resourceVersion":"1", + "annotations":{"meta.helm.sh/release-name":"kars","meta.helm.sh/release-namespace":"kars-system"}}}), + ("GET","/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/sre")=>state.sandbox.clone(), + ("GET","/api/v1/namespaces/kars-sre")=>state.namespace.clone(), + ("GET","/api/v1/namespaces/kars-sre/secrets")=>json!({ + "apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{}, + "items":state.objects.iter().filter(|(path,_)|path.starts_with("/api/v1/namespaces/kars-sre/secrets/")) + .map(|(_,object)|json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":object["metadata"]})) + .collect::>()}), + ("GET","/apis/rbac.authorization.k8s.io/v1/clusterrolebindings")=>json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBindingList","metadata":{}, + "items":std::iter::once(state.binding.clone()).chain(state.objects.values() + .filter(|object|object["kind"]=="ClusterRoleBinding").cloned()).collect::>()}), + ("GET","/apis/rbac.authorization.k8s.io/v1/rolebindings")=>json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBindingList","metadata":{}, + "items":state.objects.values().filter(|object|object["kind"]=="RoleBinding").cloned().collect::>()}), + ("PATCH","/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/legacy")=>{ + if body["metadata"]["uid"]!=state.binding["metadata"]["uid"] {return api_error(422)} + if body["metadata"]["resourceVersion"]!=state.binding["metadata"]["resourceVersion"] {return api_error(409)} + let value=patched_binding(&state.binding,&body,dry_run); + if !dry_run {state.binding=value.clone();} + value + } + ("POST","/apis/authorization.k8s.io/v1/subjectaccessreviews")=>json!({ + "apiVersion":"authorization.k8s.io/v1","kind":"SubjectAccessReview", + "spec":body["spec"],"status":{"allowed":state.allowed || state.watch_allowed.as_ref().is_some_and(|(namespace,name)| + body["spec"]["resourceAttributes"]["verb"]=="watch" + && body["spec"]["resourceAttributes"]["namespace"].as_str()==namespace.as_deref() + && body["spec"]["resourceAttributes"]["name"].as_str()==name.as_deref())}}), + ("PATCH","/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical/status")=>{ + let mut reg=state.objects.get("/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical") + .cloned().unwrap_or_else(||serde_json::to_value(registration()).unwrap()); + reg["status"]=body["status"].clone(); + reg["metadata"]["resourceVersion"]="2".into(); + state.objects.insert("/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical".into(),reg.clone()); + reg + } + ("POST","/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token")=>json!({ + "apiVersion":"authentication.k8s.io/v1","kind":"TokenRequest","metadata":{},"spec":body["spec"], + "status":{"token":"PRIVATE_KUBE_TOKEN","expirationTimestamp":(chrono::Utc::now()+chrono::Duration::hours(1)).to_rfc3339()}}), + _=>return api_error(404), + }; + ResponseTemplate::new(200).set_body_json(value) + }).mount(&server).await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let client = Client::try_from(kube::Config::new(server.uri().parse().unwrap())).unwrap(); + (server, client, state) +} + +#[tokio::test] +async fn canonical_source_and_namespace_uids_authorize_not_labels_or_names() { + let (_server, client, state) = fixture().await; + let reg = registration(); + assert!(live::verify(&client, ®).await.is_ok()); + for target in ["source", "namespace", "foreign-workspace", "controller"] { + let mut reg = registration(); + match target { + "source" => reg.spec.sandbox.uid = "recreated".into(), + "namespace" => reg.spec.runtime_namespace.uid = "recreated".into(), + "controller" => reg.spec.controller.deployment.uid = "recreated".into(), + _ => { + state.lock().unwrap().namespace["metadata"]["annotations"]["kars.azure.com/sandbox-namespace"] = + "untrusted".into() + } + } + assert!(live::verify(&client, ®).await.is_err(), "{target}"); + } + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn reviewed_retirement_preserves_other_subjects_and_has_cas_preconditions() { + let (_server, client, state) = fixture().await; + let reg = registration(); + let reviewed = bindings::review(&client, ®).await.unwrap(); + bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap(); + let state = state.lock().unwrap(); + assert_eq!( + state.binding["subjects"], + json!([{"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]) + ); + let patch = &state + .calls + .iter() + .find(|(method, _, _)| method == "PATCH") + .unwrap() + .2; + assert_eq!(patch["metadata"]["uid"], "binding"); + assert_eq!(patch["metadata"]["resourceVersion"], "1"); +} + +#[tokio::test] +async fn unreviewed_or_replaced_bindings_stop_before_mutation() { + let (_server, client, state) = fixture().await; + for changed in ["missing-review", "uid", "version"] { + let mut reg = registration(); + match changed { + "missing-review" => reg.spec.legacy_bindings.clear(), + "uid" => reg.spec.legacy_bindings[0].uid = "other".into(), + _ => reg.spec.legacy_bindings[0].resource_version = "other".into(), + } + assert!(bindings::review(&client, ®).await.is_err()); + } + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn real_authorization_review_denial_is_required_and_errors_do_not_hide_access() { + let (_server, client, state) = fixture().await; + check_secret_denial(&client, "kars-sre").await.unwrap(); + assert_eq!( + state + .lock() + .unwrap() + .calls + .iter() + .filter(|(method, _, _)| method == "POST") + .count(), + 18 + ); + state.lock().unwrap().allowed = true; + assert!(check_secret_denial(&client, "kars-sre").await.is_err()); + state.lock().unwrap().api_error = Some(403); + let error = check_secret_denial(&client, "kars-sre").await.unwrap_err(); + assert!(!error.contains("PRIVATE_SENTINEL")); +} + +#[test] +fn proxy_projection_preserves_azure_identity_and_pinned_agent_image() { + let projection = pod::Projection { + epoch: "epoch".into(), + registration_uid: "registration".into(), + tls_expiry: "expiry".into(), + credential_uid: "credential-uid".into(), + }; + let mut spec = json!({"serviceAccountName":"sandbox","volumes":[],"containers":[ + {"name":"agent","image":"customer/hermes:pinned","env":[],"volumeMounts":[]}, + {"name":"inference-router","env":[],"volumeMounts":[]} ]}); + pod::project(&mut spec); + assert_eq!(spec["serviceAccountName"], "sandbox"); + assert_eq!(spec["containers"][0]["image"], "customer/hermes:pinned"); + assert_eq!(spec["automountServiceAccountToken"], false); + let agent = serde_json::to_string(&spec["containers"][0]).unwrap(); + assert!(!agent.contains(PRIVATE_SECRET)); + assert!(!agent.contains("router-kubernetes")); + assert!(agent.contains("sre-api-agent")); + assert!(agent.contains("127.0.0.1")); + assert!(agent.contains("9446")); + assert_eq!( + pod::annotations(&projection, "agent")["azure.workload.identity/skip-containers"], + "agent,egress-guard" + ); +} + +#[test] +fn registration_is_cluster_scoped_and_requires_complete_exact_identity() { + use kube::CustomResourceExt; + let crd = KarsSRERegistration::crd(); + assert_eq!(crd.spec.scope, "Cluster"); + let mut reg = registration(); + reg.validate().unwrap(); + reg.spec.sandbox.namespace = "untrusted".into(); + assert!(reg.validate().is_err()); + reg = registration(); + reg.metadata.uid = None; + assert!(reg.validate().is_err()); +} + +#[tokio::test] +async fn admission_requires_observed_enforced_policies_without_type_warnings() { + let (_server, client, state) = fixture().await; + let path = "/apis/admissionregistration.k8s.io/v1"; + for name in admission::POLICIES { + state.lock().unwrap().objects.insert(format!("{path}/validatingadmissionpolicies/{name}"),json!({ + "metadata":{"name":name,"generation":2},"spec":{"failurePolicy":"Fail","validations":[]}, + "status":{"observedGeneration":2,"typeChecking":{}}})); + state.lock().unwrap().objects.insert(format!("{path}/validatingadmissionpolicybindings/{name}"),json!({ + "metadata":{"name":name},"spec":{"policyName":name,"validationActions":["Deny","Audit"]}})); + } + admission::verify(&client).await.unwrap(); + let policy = format!( + "{path}/validatingadmissionpolicies/{}", + admission::POLICIES[0] + ); + state.lock().unwrap().objects.get_mut(&policy).unwrap()["status"]["typeChecking"] = json!({"expressionWarnings":[{"fieldRef":"spec.validations[0].expression","warning":"undeclared field"}]}); + assert!(admission::verify(&client).await.is_err()); + state.lock().unwrap().objects.get_mut(&policy).unwrap()["status"] = + json!({"observedGeneration":1,"typeChecking":{}}); + assert!(admission::verify(&client).await.is_err()); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn credentials_are_private_bound_and_renewable_without_agent_jwt() { + let (_server, client, state) = fixture().await; + let reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let private_path = "/api/v1/namespaces/kars-sre/secrets/sre-api-router-identity"; + let agent_path = "/api/v1/namespaces/kars-sre/secrets/sre-api-agent"; + let decode = |value: &Value, key: &str| { + String::from_utf8( + base64::engine::general_purpose::STANDARD + .decode(value["data"][key].as_str().unwrap()) + .unwrap(), + ) + .unwrap() + }; + let before = { + let locked = state.lock().unwrap(); + let private = &locked.objects[private_path]; + let agent = &locked.objects[agent_path]; + assert_eq!(decode(private, "kube-token"), "PRIVATE_KUBE_TOKEN"); + assert_eq!(decode(agent, "token"), decode(private, "agent-token")); + assert_eq!(decode(agent, "ca.crt"), decode(private, "agent-ca.crt")); + assert_eq!(agent["data"].as_object().unwrap().len(), 3); + assert!(!serde_json::to_string(agent).unwrap().contains("kube-token")); + let request = locked + .calls + .iter() + .find(|(_, path, _)| path.ends_with("/token")) + .unwrap(); + assert_eq!(request.2["spec"]["boundObjectRef"]["kind"], "Secret"); + assert_eq!( + request.2["spec"]["boundObjectRef"]["uid"], + private["metadata"]["uid"] + ); + assert!(decode(private, "server-key.pem").starts_with("-----BEGIN PRIVATE KEY-----")); + private.clone() + }; + let writes = state + .lock() + .unwrap() + .calls + .iter() + .filter(|(method, _, _)| method == "PATCH") + .count(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + assert_eq!( + writes, + state + .lock() + .unwrap() + .calls + .iter() + .filter(|(method, _, _)| method == "PATCH") + .count() + ); + state.lock().unwrap().objects.get_mut(private_path).unwrap()["metadata"]["annotations"]["kars.azure.com/sre-tls-expiry"] = + "0".into(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let locked = state.lock().unwrap(); + assert_ne!( + decode(&before, "agent-token"), + decode(&locked.objects[private_path], "agent-token") + ); + assert_eq!( + decode(&locked.objects[private_path], "agent-token"), + decode(&locked.objects[agent_path], "token") + ); +} + +#[tokio::test] +async fn unsafe_authorization_or_recreated_targets_never_receive_private_material() { + for problem in [ + "secret-access", + "source-recreated", + "namespace-recreated", + "foreign-secret", + "api-error", + ] { + let (_server, client, state) = fixture().await; + let reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + { + let mut locked = state.lock().unwrap(); + locked.calls.clear(); + match problem { + "secret-access" => locked.allowed = true, + "source-recreated" => locked.sandbox["metadata"]["uid"] = "other".into(), + "namespace-recreated" => locked.namespace["metadata"]["uid"] = "other".into(), + "foreign-secret" => { + locked.objects.insert("/api/v1/namespaces/kars-sre/secrets/sre-api-router-identity".into(), + json!({"apiVersion":"v1","kind":"Secret","metadata":{"name":"sre-api-router-identity","namespace":"kars-sre","uid":"foreign","resourceVersion":"1"}})); + } + _ => locked.api_error = Some(403), + } + } + let error = credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap_err(); + assert!(!error.contains("PRIVATE_SENTINEL")); + assert!(state.lock().unwrap().calls.iter().all(|(method,path,_)| + method=="GET" || path.ends_with("/subjectaccessreviews")), "{problem}"); + } +} + +#[tokio::test] +async fn migration_waits_for_consumers_and_retires_only_owned_private_material() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.legacy_consumer = Some(ConsumerReview { + namespace: RUNTIME_NAMESPACE.into(), + name: "sre".into(), + uid: "consumer".into(), + resource_version: "1".into(), + }); + state.lock().unwrap().objects.insert("/apis/apps/v1/namespaces/kars-sre/deployments/sre".into(), + json!({"metadata":{"name":"sre","namespace":"kars-sre","uid":"consumer","resourceVersion":"1"}, + "spec":{"replicas":0,"selector":{"matchLabels":{"app":"sre"}},"template":{"metadata":{},"spec":{"containers":[]}}}})); + state.lock().unwrap().objects.insert( + "/api/v1/namespaces/kars-sre/pods".into(), + json!({"kind":"PodList","apiVersion":"v1","metadata":{},"items":[{"metadata":{"name":"old"}, + "spec":{"containers":[],"serviceAccountName":"sandbox"}}]}), + ); + assert_eq!( + migration::stop_legacy_consumer(&client, ®) + .await + .unwrap_err(), + migration::WAITING_FOR_CONSUMERS + ); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + credentials::retire(&client, ®).await.unwrap(); + let locked = state.lock().unwrap(); + assert!( + !locked + .objects + .contains_key("/api/v1/namespaces/kars-sre/secrets/sre-api-router-identity") + ); + assert!( + locked + .calls + .iter() + .filter(|(method, _, _)| method == "DELETE") + .all( + |(_, _, body)| body["preconditions"]["uid"].as_str().is_some() + && body["preconditions"]["resourceVersion"].as_str().is_some() + ) + ); +} + +#[tokio::test] +async fn old_control_token_consumers_must_disappear_even_after_rollout_counters_converge() { + let (_server, client, state) = fixture().await; + let reg = registration(); + let epoch = reg.epoch(); + { + let mut locked = state.lock().unwrap(); + locked.objects.insert("/api/v1/secrets".into(),json!({ + "kind":"SecretList","apiVersion":"v1","metadata":{},"items":[{ + "kind":"Secret","apiVersion":"v1","metadata":{"name":"router-services-admin","namespace":"kars-sre","uid":"control","resourceVersion":"1", + "labels":{"app.kubernetes.io/managed-by":"kars-controller"}, + "annotations":{"kars.azure.com/sandbox-uid":"source","kars.azure.com/namespace-uid":"runtime-ns",EPOCH:epoch}}}]})); + locked.objects.insert("/apis/apps/v1/namespaces/kars-sre/deployments/sre".into(),json!({ + "metadata":{"name":"sre","namespace":"kars-sre","uid":"consumer","resourceVersion":"1","generation":2, + "labels":{"kars.azure.com/sandbox":"sre"}, + "managedFields":[{"manager":crate::field_managers::CLAWSANDBOX,"operation":"Apply","apiVersion":"apps/v1", + "fieldsType":"FieldsV1","fieldsV1":{"f:spec":{}}}]}, + "spec":{"replicas":0,"selector":{"matchLabels":{"app":"sre"}},"template":{"metadata":{"annotations":{EPOCH:epoch}}, + "spec":{"containers":[]}}}, + "status":{"observedGeneration":2,"updatedReplicas":0,"availableReplicas":0}})); + locked.objects.insert( + "/api/v1/namespaces/kars-sre/pods".into(), + json!({ + "kind":"PodList","apiVersion":"v1","metadata":{},"items":[{"metadata":{"name":"old-router", + "deletionTimestamp":"2026-09-08T00:00:00Z"},"spec":{"containers":[]}}]}), + ); + } + let error = migration::rotate_owned_control_credentials(&client, ®) + .await + .unwrap_err(); + assert!(migration::is_waiting(&error)); + state + .lock() + .unwrap() + .objects + .get_mut("/api/v1/namespaces/kars-sre/pods") + .unwrap()["items"] = json!([]); + migration::rotate_owned_control_credentials(&client, ®) + .await + .unwrap(); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, path, _)| method == "GET" || path.ends_with("/subjectaccessreviews")) + ); +} diff --git a/controller/src/sre_registration.rs b/controller/src/sre_registration.rs new file mode 100644 index 000000000..13b552b2f --- /dev/null +++ b/controller/src/sre_registration.rs @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Cluster-scoped operator authorization for the canonical SRE identity. +//! Namespace ownership identifies an occupant; only this resource delegates +//! privileged SRE authority to that exact occupant. + +use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub const NAME: &str = "canonical"; +pub const ROUTER_SA: &str = "sre-api-router"; +pub const RUNTIME_NAMESPACE: &str = "kars-sre"; +pub const PRIVATE_SECRET: &str = "sre-api-router-identity"; +pub const AGENT_SECRET: &str = "sre-api-agent"; +pub const OWNER: &str = "kars.azure.com/sre-registration-uid"; +pub const EPOCH: &str = "kars.azure.com/sre-privacy-epoch"; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct NamedUid { + pub name: String, + pub uid: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Source { + pub namespace: String, + pub name: String, + pub uid: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ControllerIdentity { + pub namespace: NamedUid, + pub deployment: NamedUid, + pub release: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct BindingReview { + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, + pub name: String, + pub uid: String, + pub resource_version: String, + pub role_ref: k8s_openapi::api::rbac::v1::RoleRef, + pub subjects: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ConsumerReview { + pub namespace: String, + pub name: String, + pub uid: String, + pub resource_version: String, +} + +#[derive(CustomResource, Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "kars.azure.com", + version = "v1alpha1", + kind = "KarsSRERegistration", + plural = "karssreregistrations", + status = "RegistrationStatus", + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"# +)] +#[serde(rename_all = "camelCase")] +pub struct KarsSRERegistrationSpec { + pub controller: ControllerIdentity, + pub sandbox: Source, + pub runtime_namespace: NamedUid, + #[serde(default)] + pub legacy_bindings: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub legacy_consumer: Option, + #[serde(default = "enabled")] + pub enabled: bool, +} + +fn enabled() -> bool { + true +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RegistrationStatus { + pub phase: String, + pub observed_generation: i64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub privacy_epoch: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_service_account_uid: Option, + #[serde(default)] + pub legacy_secret_access_denied: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub privacy_revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +impl KarsSRERegistration { + pub fn validate(&self) -> Result<(), String> { + let label = |value: &str| { + !value.is_empty() + && value.len() <= 63 + && value + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + && value.as_bytes()[0].is_ascii_alphanumeric() + && value.as_bytes()[value.len() - 1].is_ascii_alphanumeric() + }; + if self.metadata.name.as_deref() != Some(NAME) + || self.spec.sandbox.name != "sre" + || self.spec.runtime_namespace.name != RUNTIME_NAMESPACE + || !label(&self.spec.sandbox.namespace) + || self.spec.sandbox.namespace != self.spec.controller.namespace.name + || self.spec.controller.deployment.name != "kars-controller" + || self.spec.controller.release.is_empty() + { + return Err( + "Registration must pin the canonical SRE and its controller/release namespace" + .into(), + ); + } + for uid in [ + self.metadata.uid.as_deref().unwrap_or_default(), + &self.spec.controller.namespace.uid, + &self.spec.controller.deployment.uid, + &self.spec.sandbox.uid, + &self.spec.runtime_namespace.uid, + ] { + if uid.trim().is_empty() { + return Err("Registration identity omitted an exact UID".into()); + } + } + if self.metadata.deletion_timestamp.is_some() { + return Err( + "Registration is terminating; disable and retire it before deletion".into(), + ); + } + let mut bindings = std::collections::BTreeSet::new(); + for review in &self.spec.legacy_bindings { + if !matches!(review.kind.as_str(), "ClusterRoleBinding" | "RoleBinding") + || review.name.is_empty() + || review.uid.is_empty() + || review.resource_version.is_empty() + || (review.kind == "RoleBinding") + != review.namespace.as_ref().is_some_and(|ns| label(ns)) + || !bindings.insert(( + review.kind.clone(), + review.namespace.clone(), + review.name.clone(), + )) + { + return Err( + "Legacy binding reviews require unique exact names, UIDs and resourceVersions" + .into(), + ); + } + } + if let Some(consumer) = &self.spec.legacy_consumer + && (consumer.namespace != RUNTIME_NAMESPACE + || consumer.name != "sre" + || consumer.uid.is_empty() + || consumer.resource_version.is_empty()) + { + return Err( + "Legacy SRE consumer review is incomplete or targets another workload".into(), + ); + } + Ok(()) + } + + pub fn epoch(&self) -> String { + let mut value = serde_json::json!({ + "domain":crate::sre_privacy::REVISION, + "uid":self.metadata.uid, "generation":self.metadata.generation, "spec":self.spec, + }); + value.sort_all_objects(); + crate::providers::signing::sha256_hex( + &serde_json::to_vec(&value).expect("registration serializes"), + ) + } +} diff --git a/deploy/helm/kars/templates/crd-karssreaction.yaml b/deploy/helm/kars/templates/crd-karssreaction.yaml index aba25b887..13b8abe13 100644 --- a/deploy/helm/kars/templates/crd-karssreaction.yaml +++ b/deploy/helm/kars/templates/crd-karssreaction.yaml @@ -56,7 +56,6 @@ spec: free-form params (validated per-type at reconcile time). properties: params: - additionalProperties: true description: |- Per-type params. Stored as a string-keyed map so the CRD schema emits a concrete `type: object` (apiserver rejects fields with @@ -70,6 +69,7 @@ spec: - RolloutRestart: {namespace, kind, name} - DeletePod: {namespace, name} type: object + x-kubernetes-preserve-unknown-fields: true type: description: |- Action type from the closed set (`DeleteResourceQuota`, diff --git a/deploy/helm/kars/templates/crd-karssreregistration.yaml b/deploy/helm/kars/templates/crd-karssreregistration.yaml new file mode 100644 index 000000000..1f4b1add8 --- /dev/null +++ b/deploy/helm/kars/templates/crd-karssreregistration.yaml @@ -0,0 +1,139 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: karssreregistrations.kars.azure.com + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: crd + annotations: + helm.sh/resource-policy: keep +spec: + group: kars.azure.com + scope: Cluster + names: + kind: KarsSRERegistration + plural: karssreregistrations + singular: karssreregistration + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Phase + type: string + jsonPath: .status.phase + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + x-kubernetes-validations: + - rule: "self.metadata.name == 'canonical'" + message: "The canonical SRE registration is the only supported instance" + properties: + apiVersion: {type: string} + kind: {type: string} + metadata: {type: object} + spec: + type: object + required: [controller, sandbox, runtimeNamespace] + properties: + enabled: {type: boolean, default: true} + controller: + type: object + required: [namespace, deployment, release] + properties: + namespace: + type: object + required: [name, uid] + properties: + name: {type: string, minLength: 1} + uid: {type: string, minLength: 1} + deployment: + type: object + required: [name, uid] + properties: + name: {type: string, enum: [kars-controller]} + uid: {type: string, minLength: 1} + release: {type: string, minLength: 1, maxLength: 53} + sandbox: + type: object + required: [namespace, name, uid] + properties: + namespace: {type: string, minLength: 1} + name: {type: string, enum: [sre]} + uid: {type: string, minLength: 1} + runtimeNamespace: + type: object + required: [name, uid] + properties: + name: {type: string, enum: [kars-sre]} + uid: {type: string, minLength: 1} + legacyConsumer: + type: object + required: [namespace, name, uid, resourceVersion] + properties: + namespace: {type: string, enum: [kars-sre]} + name: {type: string, enum: [sre]} + uid: {type: string, minLength: 1} + resourceVersion: {type: string, minLength: 1} + legacyBindings: + type: array + maxItems: 32 + items: + type: object + required: [kind, name, uid, resourceVersion, roleRef, subjects] + properties: + kind: {type: string, enum: [ClusterRoleBinding, RoleBinding]} + namespace: {type: string} + name: {type: string, minLength: 1} + uid: {type: string, minLength: 1} + resourceVersion: {type: string, minLength: 1} + roleRef: + type: object + required: [apiGroup, kind, name] + properties: + apiGroup: {type: string, enum: [rbac.authorization.k8s.io]} + kind: {type: string, enum: [Role, ClusterRole]} + name: {type: string, minLength: 1} + subjects: + type: array + items: + type: object + required: [kind, name] + properties: + apiGroup: {type: string} + kind: {type: string} + name: {type: string} + namespace: {type: string} + x-kubernetes-validations: + - rule: "self.sandbox.__namespace__ == self.controller.__namespace__.name" + message: "SRE must be registered in its controller/release namespace" + status: + type: object + properties: + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [type, status, reason, message, lastTransitionTime] + properties: + type: {type: string} + status: {type: string, enum: ["True", "False", "Unknown"]} + reason: {type: string} + message: {type: string} + observedGeneration: {type: integer, format: int64, minimum: 0} + lastTransitionTime: {type: string, format: date-time} + phase: {type: string} + observedGeneration: {type: integer, format: int64} + privacyEpoch: {type: string} + routerServiceAccountUid: {type: string} + legacySecretAccessDenied: {type: boolean} + privacyRevision: {type: string} + detail: {type: string} diff --git a/deploy/helm/kars/templates/sre-authority-admission.yaml b/deploy/helm/kars/templates/sre-authority-admission.yaml new file mode 100644 index 000000000..a480448d2 --- /dev/null +++ b/deploy/helm/kars/templates/sre-authority-admission.yaml @@ -0,0 +1,331 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-source-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["karssandboxes", "karssandboxes/status"] + matchConditions: + - name: reserved-sre-source + expression: >- + (object != null && (object.metadata.name == 'sre' || + (has(object.metadata.labels) && 'kars.azure.com/role' in object.metadata.labels && + object.metadata.labels['kars.azure.com/role'] == 'sre'))) || + (oldObject != null && (oldObject.metadata.name == 'sre' || + (has(oldObject.metadata.labels) && 'kars.azure.com/role' in oldObject.metadata.labels && + oldObject.metadata.labels['kars.azure.com/role'] == 'sre'))) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "Canonical SRE source changes require explicit cluster-level registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-source-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-source-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-registration-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["karssreregistrations", "karssreregistrations/status"] + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "SRE enrollment requires explicit registrar use permission" + reason: Forbidden + - expression: >- + request.operation != 'DELETE' || + (oldObject.spec.enabled == false && oldObject.?status.?phase.orValue('') == 'Retired') + message: "Disable and retire SRE authority before deleting its registration" + reason: Forbidden + - expression: >- + object == null || oldObject == null || + !has(oldObject.status) || !has(oldObject.status.routerServiceAccountUid) || + oldObject.status.phase == 'Retired' || + (object.spec.controller == oldObject.spec.controller && + object.spec.sandbox == oldObject.spec.sandbox && + object.spec.runtimeNamespace == oldObject.spec.runtimeNamespace) + message: "Retire issued SRE authority before changing its bound identities" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-registration-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-registration-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-identity + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["serviceaccounts", "serviceaccounts/token"] + matchConditions: + - name: reserved-router-identity + expression: >- + (has(request.name) && request.name == 'sre-api-router') || + (object != null && has(object.metadata) && has(object.metadata.name) && + object.metadata.name == 'sre-api-router') || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && + oldObject.metadata.name == 'sre-api-router') + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() || + (request.subResource == 'token' && + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('renew').allowed()) + message: "Reserved SRE router identity requires registrar use; its TokenRequest renewal requires explicit renew authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-identity + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-identity + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-binding-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["rbac.authorization.k8s.io"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["clusterrolebindings", "rolebindings"] + variables: + - name: oldSubjects + expression: "oldObject == null ? [] : oldObject.?subjects.orValue([])" + - name: newSubjects + expression: "object == null ? [] : object.?subjects.orValue([])" + matchConditions: + - name: sre-binding + expression: >- + (object != null && object.?subjects.orValue([]).exists(s, + (s.kind == 'ServiceAccount' && s.?namespace.orValue('') == 'kars-sre' && + s.name in ['sandbox', 'sre-api-router']) || + (s.kind == 'User' && s.name in + ['system:serviceaccount:kars-sre:sandbox', 'system:serviceaccount:kars-sre:sre-api-router']) || + (s.kind == 'Group' && s.name == 'system:serviceaccounts:kars-sre'))) || + (oldObject != null && oldObject.?subjects.orValue([]).exists(s, + (s.kind == 'ServiceAccount' && s.?namespace.orValue('') == 'kars-sre' && + s.name in ['sandbox', 'sre-api-router']) || + (s.kind == 'User' && s.name in + ['system:serviceaccount:kars-sre:sandbox', 'system:serviceaccount:kars-sre:sre-api-router']) || + (s.kind == 'Group' && s.name == 'system:serviceaccounts:kars-sre'))) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "SRE identity bindings require registrar authority" + reason: Forbidden + - expression: >- + object == null || + object.roleRef.name == 'kars-sandbox-spawner' || + !variables.newSubjects.exists(s, + (s.kind == 'ServiceAccount' && s.?namespace.orValue('') == 'kars-sre' && s.name == 'sandbox') || + (s.kind == 'User' && s.name == 'system:serviceaccount:kars-sre:sandbox') || + (s.kind == 'Group' && s.name == 'system:serviceaccounts:kars-sre')) || + (oldObject != null && object.roleRef == oldObject.roleRef && + variables.newSubjects == variables.oldSubjects) + message: "Legacy SRE agent grants may be retained unchanged for reviewed migration, never created or restored" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-binding-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-binding-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-material + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["secrets"] + matchConditions: + - name: protected-material + expression: >- + request.namespace == 'kars-sre' && + ((object != null && object.metadata.name in ['sre-api-router-identity', 'sre-api-agent']) || + (oldObject != null && oldObject.metadata.name in ['sre-api-router-identity', 'sre-api-agent'])) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "SRE identity material requires cluster registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-material + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-material + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-no-legacy-tokens + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["secrets"] + matchConditions: + - name: private-runtime + expression: "request.namespace == 'kars-sre'" + validations: + - expression: >- + !(object != null && object.?type.orValue('') == 'kubernetes.io/service-account-token' && + has(object.metadata.annotations) && + 'kubernetes.io/service-account.name' in object.metadata.annotations && + object.metadata.annotations['kubernetes.io/service-account.name'] == 'sre-api-router') && + !(oldObject != null && oldObject.?type.orValue('') == 'kubernetes.io/service-account-token' && + has(oldObject.metadata.annotations) && + 'kubernetes.io/service-account.name' in oldObject.metadata.annotations && + oldObject.metadata.annotations['kubernetes.io/service-account.name'] == 'sre-api-router') + message: "The private SRE identity exclusively uses bound TokenRequests; legacy token Secrets are forbidden for every caller" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-no-legacy-tokens + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-no-legacy-tokens + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-source-retirement + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + paramKind: + apiVersion: kars.azure.com/v1alpha1 + kind: KarsSRERegistration + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["DELETE"] + resources: ["karssandboxes"] + validations: + - expression: >- + oldObject.metadata.uid != params.spec.sandbox.uid || + (params.spec.enabled == false && params.?status.?phase.orValue('') == 'Retired') + message: "Retire registered SRE authority before deleting its source" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-source-retirement + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-source-retirement + paramRef: + name: canonical + parameterNotFoundAction: Allow + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-pending-proposals + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["CREATE"] + resources: ["karssreactions"] + validations: + - expression: "object.spec.approval.state == 'Pending'" + message: "SRE actions must be created Pending; approval is a separate operator action" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-pending-proposals + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-pending-proposals + validationActions: [Deny, Audit] diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml new file mode 100644 index 000000000..583f8168b --- /dev/null +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -0,0 +1,249 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-consumer-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["deployments", "deployments/scale"] + matchConditions: + - name: canonical-runtime-consumer + expression: >- + has(request.namespace) && request.namespace == 'kars-sre' && + ((has(request.name) && request.name == 'sre') || + (object != null && has(object.metadata) && has(object.metadata.name) && object.metadata.name == 'sre') || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && oldObject.metadata.name == 'sre')) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "Canonical SRE runtime changes require registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-consumer-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-consumer-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-role-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["rbac.authorization.k8s.io"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["roles", "clusterroles"] + matchConditions: + - name: reserved-sre-role + expression: >- + (has(request.name) && request.name in + ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent']) || + (object != null && has(object.metadata) && has(object.metadata.name) && object.metadata.name in + ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent']) || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && oldObject.metadata.name in + ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent']) || + (has(request.namespace) && request.namespace == 'kars-sre' && + ((has(request.name) && request.name == 'sre-api-self-renew') || + (object != null && has(object.metadata) && has(object.metadata.name) && + object.metadata.name == 'sre-api-self-renew') || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && + oldObject.metadata.name == 'sre-api-self-renew'))) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "Reserved SRE authority roles require explicit registrar use permission" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-role-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-role-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-mounts + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["pods", "pods/ephemeralcontainers"] + matchConditions: + - name: sre-runtime + expression: "request.namespace == 'kars-sre'" + variables: + - name: containers + expression: >- + object.spec.containers + object.spec.?initContainers.orValue([]) + + object.spec.?ephemeralContainers.orValue([]) + - name: privateMaterial + expression: >- + object.spec.?volumes.orValue([]).exists(v, + (has(v.secret) && v.secret.secretName == 'sre-api-router-identity') || + (has(v.projected) && v.projected.sources.exists(s, + has(s.secret) && s.secret.name == 'sre-api-router-identity'))) || + variables.containers.exists(c, + c.?envFrom.orValue([]).exists(e, has(e.secretRef) && e.secretRef.name == 'sre-api-router-identity') || + c.?env.orValue([]).exists(e, has(e.valueFrom) && has(e.valueFrom.secretKeyRef) && + e.valueFrom.secretKeyRef.name == 'sre-api-router-identity')) || + object.spec.?serviceAccountName.orValue('') == 'sre-api-router' + validations: + - expression: >- + !variables.privateMaterial || + authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() || + authorizer.group('').resource('pods').check('create').allowed() + message: "Private SRE material may only be mounted by registrars or cluster-wide workload controllers" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-mounts + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-mounts + validationActions: [Deny, Audit] +--- +# Pod admission must also validate parent templates: otherwise a namespaced +# caller could launder a private mount through a privileged workload controller. +{{ range $kind := list "workloads" "cronjobs" }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-{{ $kind }} + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + {{- if eq $kind "workloads" }} + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["replicationcontrollers"] + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] + - apiGroups: ["batch"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["jobs"] + {{- else }} + - apiGroups: ["batch"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["cronjobs"] + {{- end }} + matchConditions: + - name: sre-runtime + expression: "request.namespace == 'kars-sre'" + {{- if eq $kind "workloads" }} + - name: replication-controller-template-present + expression: "object.kind != 'ReplicationController' || has(object.spec.template)" + {{- end }} + variables: + - name: pod + expression: {{ if eq $kind "cronjobs" }}"object.spec.jobTemplate.spec.template.spec"{{ else }}"object.spec.template.spec"{{ end }} + - name: containers + expression: "variables.pod.containers + variables.pod.?initContainers.orValue([])" + - name: privateMaterial + expression: >- + variables.pod.?volumes.orValue([]).exists(v, + (has(v.secret) && v.secret.secretName == 'sre-api-router-identity') || + (has(v.projected) && v.projected.sources.exists(s, + has(s.secret) && s.secret.name == 'sre-api-router-identity'))) || + variables.containers.exists(c, + c.?envFrom.orValue([]).exists(e, has(e.secretRef) && e.secretRef.name == 'sre-api-router-identity') || + c.?env.orValue([]).exists(e, has(e.valueFrom) && has(e.valueFrom.secretKeyRef) && + e.valueFrom.secretKeyRef.name == 'sre-api-router-identity')) || + variables.pod.?serviceAccountName.orValue('') == 'sre-api-router' + validations: + - expression: >- + !variables.privateMaterial || + authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() || + authorizer.group('').resource('pods').check('create').allowed() + {{- if eq $kind "workloads" }} + || (request.resource.group == 'apps' && request.resource.resource == 'replicasets' && + authorizer.group('apps').resource('replicasets').check('create').allowed()) + {{- end }} + message: "Private SRE workload templates require registrar or cluster-wide workload-controller authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-{{ $kind }} + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-{{ $kind }} + validationActions: [Deny, Audit] +{{ end }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-connect + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CONNECT"] + resources: ["pods/exec", "pods/attach", "pods/portforward", "pods/proxy"] + matchConditions: + - name: sre-runtime + expression: "request.namespace == 'kars-sre'" + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() + message: "Connecting into the SRE private-identity runtime requires registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-connect + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-connect + validationActions: [Deny, Audit] diff --git a/deploy/helm/kars/templates/sre-authority-rbac.yaml b/deploy/helm/kars/templates/sre-authority-rbac.yaml new file mode 100644 index 000000000..c3580fb6d --- /dev/null +++ b/deploy/helm/kars/templates/sre-authority-rbac.yaml @@ -0,0 +1,142 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-registrar + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + resourceNames: ["canonical"] + verbs: ["use", "update", "patch", "delete"] +--- +# Deliberately unbound. Only a cluster administrator delegates registrar power. +# Controller read/status/use permissions do not permit it to enroll a tenant. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-router-renew + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + resourceNames: ["canonical"] + verbs: ["get", "renew"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-retired-agent + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: [] +--- +# Unbound in the chart. The registration controller grants this only to the +# exact claimed runtime namespace's protected, UID-owned router identity. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-private-diagnostics + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssandboxes", "inferencepolicies", "toolpolicies", "egressapprovals", + "karsmemories", "karsevals", "trustgraphs", "karspairings", "a2aagents", "mcpservers", + "karsauthconfigs", "karstasks", "karsteams", "karsprofiles", "karsskills", "karsreceipts", "karsapprovals"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods", "pods/log", "services", "configmaps", "events", "namespaces", + "serviceaccounts", "nodes", "endpoints", "resourcequotas", "limitranges", "secrets"] + verbs: ["get", "list"] + - apiGroups: ["apps"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] + verbs: ["get", "list"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies", "ingresses"] + verbs: ["get", "list"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["get", "list"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-authority-controller + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + verbs: ["get", "list", "watch"] + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + resourceNames: ["canonical"] + verbs: ["use"] + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations/status"] + resourceNames: ["canonical"] + verbs: ["get", "patch", "update"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "roles", "rolebindings"] + verbs: ["get", "list", "watch"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["create", "patch", "update", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + resourceNames: ["kars-sre-private-diagnostics", "kars-sre-action-author", "kars-sre-router-renew", "kars-sre-reader"] + verbs: ["bind"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["kubernetes"] + verbs: ["get"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kars-sre-authority-controller + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kars-sre-authority-controller +subjects: + - kind: ServiceAccount + name: kars-controller + namespace: {{ .Release.Namespace }} diff --git a/deploy/helm/kars/templates/sre.yaml b/deploy/helm/kars/templates/sre.yaml index 6f87944da..5fef64b6d 100644 --- a/deploy/helm/kars/templates/sre.yaml +++ b/deploy/helm/kars/templates/sre.yaml @@ -1,10 +1,9 @@ {{- /* kars-sre — the built-in SRE agent (Slice 1 MVP). -Gated on `.Values.sre.enabled` (default: false). Enable via: - helm upgrade --reuse-values --set sre.enabled=true ... -or — preferred — via the CLI: - kars sre install +Gated on `.Values.sre.enabled` (default: false). `kars sre install` creates +and enrolls a fresh source. Existing installs require reviewed authority +migration; Helm/GitOps users follow docs/how-to/sre-authority.md. What this template creates (when sre.enabled=true): - InferencePolicy `sre-inference` (Release.Namespace) @@ -14,15 +13,14 @@ What this template creates (when sre.enabled=true): `sre.py` but only registers its tools when this env is set — standard Hermes sandboxes don't get the SRE tool surface) - ClusterRole `kars-sre-reader` — kars-CR read scope (Slice 1) - - ClusterRoleBinding `kars-sre-reader` — bound to the SA - `sandbox` in namespace `kars-sre` (the controller-created default) + - No fresh privileged runtime binding. Registration reconciliation grants + only the separate router identity after verified legacy credential denial. - ToolPolicy `sre-tools` (Release.Namespace) — gates the sre_* tool surface Per design (docs/blueprints/07-kars-sre-proposal.md §7.8 — privilege containment): - - Sandbox uniqueness VAP (kars-sre-uniqueness) — Slice 1 ships the - label `kars.azure.com/role=sre`; the VAP itself lands in Slice 3 - alongside the typed apply-fix path + - Admission protects the registered source and private runtime identities; + labels alone never authorize SRE privileges. - kars_spawn family deregistered when KARS_SRE_ENABLED=true (enforced in the plugin __init__.py — §7.8.5) - kars_mesh_* family deregistered when KARS_SRE_ENABLED=true @@ -35,7 +33,28 @@ containment): {{- $retainNamespace := false }} {{- $legacyWriter := dict }} {{- $retainWriter := false }} +{{- $retainedBindings := list }} {{- if .Release.IsUpgrade }} +{{- range $name := list "kars-sre-reader" "kars-sre-action-author" }} +{{- $binding := lookup "rbac.authorization.k8s.io/v1" "ClusterRoleBinding" "" $name }} +{{- if $binding }} +{{- $annotations := default dict $binding.metadata.annotations }} +{{- if and (eq (index $annotations "meta.helm.sh/release-name") $.Release.Name) (eq (index $annotations "meta.helm.sh/release-namespace") $.Release.Namespace) }} +{{- range $subject := default list $binding.subjects }} +{{- if or (and (eq $subject.kind "ServiceAccount") (eq $subject.name "sandbox") (eq (default "" $subject.namespace) "kars-sre")) (and (eq $subject.kind "User") (eq $subject.name "system:serviceaccount:kars-sre:sandbox")) }} +{{- if not (($.Values.sre | default dict).authorityStage | default false) }} +{{- fail "Legacy SRE grants must be explicitly reviewed/enrolled and retired before this upgrade; run kars sre authority stage/preview/enroll/migrate" }} +{{- end }} +{{- end }} +{{- end }} +{{- $_ := set $binding.metadata "annotations" (mergeOverwrite (deepCopy $annotations) (dict "helm.sh/resource-policy" "keep")) }} +{{- $_ := unset $binding.metadata "managedFields" }} +{{- $retainedBindings = append $retainedBindings $binding }} +{{- else if ($.Values.sre | default dict).enabled }} +{{- fail "An existing SRE binding belongs to another release; no adoption or privilege grant is permitted" }} +{{- end }} +{{- end }} +{{- end }} {{- $legacyNamespace = lookup "v1" "Namespace" "" "kars-sre" }} {{- if $legacyNamespace }} {{- $annotations := default dict $legacyNamespace.metadata.annotations }} @@ -53,6 +72,12 @@ containment): {{- end }} {{- end }} {{- end }} +{{- range $binding := $retainedBindings }} +--- +# Keep the reviewed retired binding, preserving unrelated subjects and CAS +# identity. No fresh SRE privilege binding is emitted by Helm. +{{ toYaml $binding }} +{{- end }} {{- if $retainNamespace }} --- # Retain an earlier release's Namespace even when disabling SRE. Dropping it @@ -112,15 +137,27 @@ spec: dailyTokens: {{ (.Values.sre | default dict).dailyTokens | default 2000000 }} --- # kars-sre KarsSandbox — Hermes runtime, SRE plugin gated on env. +{{- if ((.Values.sre | default dict).authorityStage | default false) }} +{{- $source := lookup "kars.azure.com/v1alpha1" "KarsSandbox" .Release.Namespace "sre" }} +{{- if not $source }} +{{- fail "Authority staging cannot CREATE an enabled SRE source by name; stage disabled core, then use atomic authority stage-source" }} +{{- end }} +{{- $sourceAnnotations := default dict $source.metadata.annotations }} +{{- if not (and (eq (index $sourceAnnotations "meta.helm.sh/release-name") .Release.Name) (eq (index $sourceAnnotations "meta.helm.sh/release-namespace") .Release.Namespace)) }} +{{- fail "Authority staging cannot adopt an unowned SRE source; explicitly review its identity outside Helm" }} +{{- end }} +# Stage the controller/API without rewriting the source being reviewed. +{{- $retainedSource := omit (deepCopy $source) "status" }} +{{- $_ := unset $retainedSource.metadata "managedFields" }} +{{ toYaml $retainedSource }} +{{- else }} apiVersion: kars.azure.com/v1alpha1 kind: KarsSandbox metadata: name: sre namespace: {{ .Release.Namespace }} labels: - # The label the future kars-sre-uniqueness VAP keys on (Slice 3). - # Slice 1 ships the label so by-the-time-VAP-lands no operator can - # have applied a second role=sre sandbox first. + # Admission reserves SRE sources; this label alone never grants authority. kars.azure.com/role: sre kars.azure.com/channels: none app.kubernetes.io/name: kars @@ -194,7 +231,7 @@ spec: # registry. The SRE agent does not use the mesh (§7.8.6 — three # layers: spec, image plugin, networkPolicy; this is layer 3). allowedEndpoints: - # In-cluster apiserver — the SRE agent's primary counterparty. + # Router-only upstream; the agent uses the filtered loopback HTTPS API. - host: kubernetes.default.svc.cluster.local port: 443 # Telegram Bot API — required when the operator configures @@ -215,6 +252,7 @@ spec: port: {{ .port }} {{- end }} {{- end }} +{{- end }} --- # kars-sre ToolPolicy — gates the sre_* tool surface. # @@ -384,23 +422,6 @@ rules: # kubectl accepts CRBs that reference not-yet-existing SAs — the # binding activates when the SA appears on first sandbox # reconciliation. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kars-sre-reader - labels: - app.kubernetes.io/name: kars - app.kubernetes.io/component: sre - app.kubernetes.io/managed-by: {{ .Release.Service }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kars-sre-reader -subjects: - - kind: ServiceAccount - name: sandbox - namespace: kars-sre ---- # --------------------------------------------------------------------- # Slice 3 — Typed apply-fix path (KarsSREAction CRD + writer SA) # --------------------------------------------------------------------- @@ -487,23 +508,6 @@ rules: resources: ["karssreactions/status"] verbs: ["get"] --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kars-sre-action-author - labels: - app.kubernetes.io/name: kars - app.kubernetes.io/component: sre - app.kubernetes.io/managed-by: {{ .Release.Service }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kars-sre-action-author -subjects: - - kind: ServiceAccount - name: sandbox - namespace: kars-sre ---- # Operator-facing role. Cluster admin binds humans / groups to # this manually (e.g. # kubectl create clusterrolebinding sre-approvers \ diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md new file mode 100644 index 000000000..330c3910e --- /dev/null +++ b/docs/how-to/sre-authority.md @@ -0,0 +1,317 @@ +# Registered SRE authority and credential privacy + +SRE namespace occupancy is not authorization. The cluster-scoped +`KarsSRERegistration/canonical` is the operator trust root. It pins the exact +controller Deployment and controller namespace UIDs, owning release, canonical +`sre` Sandbox namespace/name/UID, and `kars-sre` namespace UID. + +The `kars-sre-registrar` ClusterRole is **unbound**. Only an explicit cluster +administrator delegation gives an operator registration permissions. +Namespace administrators, agents, and the Bridge BFF receive none by default. +Installers also need their normal core/Helm deployment permissions. +Kubernetes must support stable `admissionregistration.k8s.io/v1` +ValidatingAdmissionPolicy and CEL authorizer checks. The controller refuses +privilege until all fourteen policies are observed, type-checked without +warnings, and bound with `Deny`. + +## Fresh installation + +Install the core prerequisite controller and router first. SRE remains disabled +by default. A registrar can then run: + +```sh +kars sre install --namespace kars-system --release kars +``` + +The CLI stages only supporting policies and a genuinely new source CR, using +atomic CREATE and its returned UID. A racing existing CR is never fetched and +adopted. It waits for that UID's complete namespace claim and backlink before +enrollment. No privileged RoleBinding is emitted by Helm. + +If either the source or runtime namespace already exists, use explicit review +below instead. A foreign `kars-sre` occupant remains untouched and receives no +privilege. + +## Existing installation: stage, review, enroll + +Normal `up`, `up --upgrade`, `upgrade`, local-Kubernetes `dev`, and `push --apply` +stop before Kubernetes deployment changes when legacy SRE grants remain. +Staging is the explicit registrar operation that installs the prerequisite +controller/API while retaining old grants unchanged: + +```sh +kars sre authority stage --namespace kars-system --release kars \ + --controller-image : \ + --router-image : --dry-run +``` + +Review the output, then run the same command without `--dry-run`. Staging does +not enroll an occupant or issue private SRE credentials. + +SRE installation and staging inspect all Helm release statuses. Helm 3 uses +`list --all`; Helm 4 removed that flag and lists all statuses by default. The +CLI retries without it only after that exact flag error and a confirmed Helm 4 +version. Other discovery failures remain errors, not an absent release or +permission to install over existing resources. + +Staging uses Helm 3's built-in waiter (`--wait`) or Helm 4's explicit +`--wait=legacy`, with the same eight-minute bound. It waits for built-in +workloads and established APIs, **not** custom-resource `Ready`: an existing +unenrolled SRE source cannot become Ready until the subsequent enrollment. +This is not a readiness bypass. `authority migrate` and normal installation +still require the fully observed, policy-type-checked authority Ready gate. + +Before installing dependent policies, staging also checks the action CRD. +The immutable BASE365 action schema's `params.additionalProperties: true` +is incompatible with Kubernetes 1.31 CEL type checking. For that exact +recognized schema only, staging uses UID/resourceVersion-tested JSON Patch +to replace the boolean with `x-kubernetes-preserve-unknown-fields: true`. +It does not adopt the old object or rewrite its other fields or metadata. +Known API defaults are tolerated; foreign installation ownership, +custom schemas/versions/validators, replacements, and API failures stop +staging. Resolve those conflicts through explicit installation/schema review, +not force adoption. Both registration and action APIs must be established +before their dependent policies can be used. + +```sh +kars sre authority preview --namespace kars-system --release kars +kars sre authority enroll --namespace kars-system --release kars \ + --sandbox-uid --namespace-uid \ + --binding 'ClusterRoleBinding//kars-sre-reader=@' \ + --binding 'ClusterRoleBinding//kars-sre-action-author=@' \ + --consumer '@' --dry-run +``` + +Use **every** binding review printed by preview, not just the illustrative +names above. Updating an existing registration additionally requires +`--registration-uid` and `--resource-version`. After review, omit `--dry-run` +and wait for migration: + +```sh +kars sre authority migrate --namespace kars-system --release kars +``` + +The controller validates the full review set, then submits every planned +retirement PATCH to the API server with `dryRun=All` before applying any of +them. Each dry-run and real PATCH uses the identical reviewed UID/resourceVersion, +retirement annotation and surviving subjects. Only the legacy SRE subject is +removed; unrelated subjects and the binding's roleRef remain intact. The +reviewed old consumer is stopped only after all binding retirements succeed. +Broad group grants and unreviewed resources stop migration; the operator must +restructure those grants explicitly. + +### Binding authorization and custom roles + +Kubernetes applies RBAC privilege-escalation checks even to a **subtractive** +ClusterRoleBinding or RoleBinding update. Ordinary `patch` permission and +registrar `use` are not sufficient: the controller must already hold the +referenced permissions or have the appropriate named `bind` permission. +The chart grants `bind` on exactly `kars-sre-reader`, +`kars-sre-private-diagnostics`, `kars-sre-action-author` and `kars-sre-router-renew`. +It does not grant wildcard `bind`, `escalate`, cluster-admin, or automatic +authority over custom roles. + +A reviewed custom role can therefore fail server-side preflight. A failure +such as `Preflight reviewed SRE ClusterRoleBinding retirement: Kubernetes status 403` +leaves all legacy bindings and the consumer unchanged on that initial attempt. +The controller relies on the actual API server's RBAC, admission and validation +decisions; it does not simulate those checks or grant itself missing authority. +An operator must inspect the referenced role and explicitly resolve the grant: +restructure or retire it, or separately authorize only the exact required +role-specific binding permission after review. Never add broad bind/escalate +rights merely to make migration pass. Refresh enrollment UID/resourceVersion +reviews if an operator changes the reviewed resources. + +Preflight is **not a multi-resource transaction**. RBAC, admission or object +versions can change before a real PATCH. Every real write still enforces its +original UID/resourceVersion and current API authorization, and errors stop the +attempt. Earlier successful retirements can remain applied; they are recognized +on retry, not rolled back by restoring old privileges. The consumer is not +stopped and private credentials are not issued while retirement remains incomplete. + +Real Kubernetes authorization reviews must deny the old SRE principal Secret +get/list/watch access before private credentials are issued. The shared +`shared/sre_privacy.rs` contract checks both namespace and cluster scope, +including generic access and name-restricted access to the two protected +Secrets. Both issuance and each proxy authorization run these live checks; +an earlier `Ready` status boolean is not sufficient. Namespace/source +recreation, API errors, or incomplete claims fail closed. No ServiceAccount +delete/recreate shortcut revokes the legacy identity. + +The controller also rotates proven owned `router-services-admin` credentials, +if any exist, and restarts their owned consumers. Their absence on the +prerequisite base is normal. Later governed-service issuers must call +`sre_authority::privacy_epoch` immediately before issuance. + +### Legacy token Secrets and watch-only grants + +The private `sre-api-router` identity must **never** use legacy +`kubernetes.io/service-account-token` Secrets. Admission denies CREATE and +UPDATE for arbitrary Secret names when either the old or new object combines +that type with `kubernetes.io/service-account.name=sre-api-router`. There is no +registrar or Kubernetes token-controller exception. Normal bound TokenRequest +renewal remains supported. + +Before creating the private ServiceAccount, granting authority, or issuing +credentials, the controller inventories Secret **metadata only**. Prestaged +aliases are rejected even if already populated, carrying an obsolete SA UID, +or annotated as another type. A renamed alias retaining the current or +registered SA UID is also rejected. Incomplete inventories and API errors fail +closed. Unknown Secret values are never read for this inspection, logged, +adopted, or deleted; an operator must review and resolve the conflicting objects. + +Discovery after a prior `Ready` revokes owned grants, retires owned private +credentials/identity, and clears the published privacy evidence. Foreign +objects and unrelated binding subjects remain intact. The Pod's `sandbox` +ServiceAccount and Azure federated subject are never deleted or replaced. +Recovery requires clean live checks and new bound credentials; replacement of +the owned Secret UID invalidates tokens from a previous privacy epoch. That UID +also drives consumer rollout, independently of TLS expiry. + +Privacy revision `kars.azure.com/sre-privacy/v2` prevents an old get/list-only +status from authorizing the new proxy or issuance helper. Watch-only and +wildcard Secret grants are included in controller and CLI legacy inventory; +unreviewed/group grants block migration before private issuance. The router +identity alone receives narrowly scoped authorization-review creation +permission for the live check. This API is not exposed through the agent proxy. + +## Helm and GitOps sequencing + +1. Install the prerequisite core/chart with `sre.enabled=false` for a fresh + installation. For an existing Helm-owned enabled SRE, use the explicit + `authority stage` workflow above: server-side lookup retains the exact + legacy source and grant UIDs/resourceVersions rather than re-rendering or + replacing them. Do not use offline `helm template` to guess those identities. +2. Wait for the CRD and admission policies to converge. Delegate the unbound + registrar role explicitly to the operator performing enrollment. Do not + grant it to the tenant GitOps controller or SRE agent. +3. For a fresh source, run `kars sre authority stage-source`, which atomically + CREATEs the CR and reports its actual source/namespace UIDs. For an existing + source, run `authority preview` instead; no source adoption by name occurs. +4. Review the `authority enroll --dry-run` JSON and exact binding/consumer + reviews. CREATE that cluster-scoped registration under registrar authority, + or apply an explicitly reviewed UID/resourceVersion-fenced update. The CLI + performs these operations; a GitOps operator may commit the same reviewed + specification but must not omit the API identity checks when applying it. +5. Wait for `Ready` with matching `observedGeneration`, then enable + `sre.enabled=true,sre.authorityStage=false` in the managed release. + GitOps pruning must preserve the chart's retained authority, registration, + legacy-binding, and namespace resources through this sequence. + +Non-Helm/template installs use the same CLI stage/enroll workflow; staging +validates all authority resources first and CAS-updates only controller image +and router image configuration, retaining unrelated settings. Different +unowned authority objects require explicit operator resolution, not force +adoption. Standard install/upgrade commands are not a migration bypass. + +### Azure resource-group teardown + +`kars destroy --all --yes --resource-group [--subscription ]` +captures one Azure subscription ID and pins every inventory, credential, +delete, and purge command to it. It checks **every** AKS cluster in the actual +resource group, using that cluster's ARM-issued credentials, not the global +current Kubernetes context. Credentials are written only to exclusively +created private files (0700 directory, 0600 files) in the current directory +and removed afterward; the global kubeconfig is never changed. + +An explicit `--context` must match one of those clusters by the ARM-provided +TLS endpoint/CA and live `kube-system` UID. This does not exempt other clusters +from retirement checks. ARM IDs and immutable AKS `resourceUid` values are +inventoried again before deletion; changed/ambiguous inventory, missing +identity evidence, inaccessible APIs, and unretired authority all block +deletion. A proven empty AKS inventory is allowed. `--all --local` is rejected +because `--all` deletes Azure resources, irrespective of the sandbox name. + +Preflight and Azure group deletion are not an atomic transaction. As with +other administrative teardown, stop concurrent provisioning or re-enrollment +in the group while performing teardown. No missing API access is interpreted +as retirement or absence. + +## Existing Hermes images remain compatible + +The Pod still uses ServiceAccount `sandbox`, preserving its Azure Workload +Identity federated subject and existing IMDS behavior. The router alone receives +the ordinary projected Kubernetes credential. A separate `sre-api-router` +identity, scoped to the registered namespace incarnation, performs diagnostic +API access and renews its own Secret-bound short-lived TokenRequests. + +The agent receives **no Kubernetes JWT**. Its existing paths contain an opaque, +agent-safe proxy credential, loopback CA, and namespace: + +- `/var/run/secrets/kubernetes.io/serviceaccount/token` +- `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` +- `/var/run/secrets/kubernetes.io/serviceaccount/namespace` + +Standard `KUBERNETES_SERVICE_HOST/PORT` point to `https://127.0.0.1:9446`. +Pinned Hermes clients continue using HTTPS, CA verification, raw pod-log GETs, +and proposal POSTs without an image-specific fallback. Azure token projection +is excluded from the agent container. The old apiserver egress bypass is gone. +Upstream Pod-log requests use API-compatible media negotiation; the facade +still returns only the bounded plain-text log response. +Admission protects both direct Pod mounts and Deployment/ReplicaSet/Job and +CronJob templates from laundering a private mount through Kubernetes workload +controllers. Exec/attach/port-forward into the private SRE runtime requires +registrar authority. Cluster-wide workload controllers remain trusted; +installing a custom privileged controller is a cluster-operator action. +The Deployment-controller handoff is authorized only for ReplicaSet requests +and requires cluster-wide `apps/replicasets` CREATE authority; namespaced +workload permissions are insufficient. It does not grant the Deployment +controller Pod CREATE or registrar authority. + +The registered router's API egress includes the canonical Service IP and its +ready HTTPS endpoint IP/port pairs, since policy enforcement can see the +post-DNAT destination rather than the Service VIP. These are exact `/32` or +`/128` targets, not private-subnet allowances; malformed, missing, terminating, +or oversized endpoint inventories fail closed. The controller fetches +`default/kubernetes`, with only named `kubernetes` Endpoints GET added to its +authority role, and refreshes registered SRE reconciliation every 30 seconds. +The UID-1000 egress guard and opaque agent credential are unchanged. + +The proxy checks current registration and live UID/claim authority. It permits +the bounded first-party diagnostic read/log/metrics paths and Pending-only +`KarsSREAction` creation in `kars-sre`. Secret responses retain key names but +exclude values, `stringData`, annotations, labels, and managed-field copies. +The typed `SecretList` envelope supplies item types when Kubernetes omits item +`kind`/`apiVersion`; conflicting explicit type metadata is still rejected. +Encoded/noncanonical paths, watches, streaming log follow, token requests, +exec/proxy subresources, arbitrary writes, and non-JSON media escapes are denied. +The existing Hermes proposal builder's two diagnostic labels are accepted; +owner references, status injection, arbitrary metadata, and approval fields +other than exactly `{"state":"Pending"}` are rejected. + +Limits: 16 concurrent operations, 64 KiB proposal bodies, 8 MiB JSON responses, +and 256 KiB logs. First-party watchers poll rather than stream. Private +credential renewal never falls back to the agent or Pod's less-privileged +credential. TLS rotation changes the opaque token and CA together and rolls +the SRE consumer; the old client rebuilds its TLS client on token change. + +## Retirement and rollback + +```sh +kars sre authority retire --registration-uid --resource-version +kars sre uninstall --namespace kars-system --release kars +``` + +Retirement revokes owned private grants and credentials before source cleanup. +The registrar-authorized controller first quiesces and UID/resourceVersion- +fences deletion of its owned SRE Deployment; it does not leave that protected +object for the unprivileged namespace controller. Foreign/replaced consumers +and external finalizers are preserved, not adopted or forced. Retained Retired +records can finish this owned cleanup without reissuing authority. +Name-sensitive admission also inspects the actual object/oldObject name when +collection DELETE omits `request.name`: ordinary collection cleanup remains +valid, while protected items still require registrar authority. +Uninstall/destroy refuse an active enrollment or unretired legacy grants. +Retired registrations remain audit records; recreating the source requires +explicit enrollment of the new UIDs (`authority stage-source` can atomically +stage a genuinely new source). + +Do not roll back to a release that restores agent-held Kubernetes credentials +or legacy SRE grants. CLI rollback is rejected while registration exists, and +retained admission policies reject restoration of the old privileged subject. +Use a reviewed roll-forward release. Do not prune retained authority/admission +objects through GitOps without an operator-reviewed retirement. + +This boundary assumes the controller namespace and runtime infrastructure are +operator controlled. Cluster administrators can change RBAC/admission itself; +deliberately bypassing those controls is not defended by a container mount. diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md new file mode 100644 index 000000000..636a3342b --- /dev/null +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -0,0 +1,382 @@ +# Security audit — registered SRE credential authority + +Status: **source audit approved under explicit maintainer delegation**. +Integration remains conditional on every required exact-head technical check. +This approval does not authorize a customer deployment or a change to `main`. + +## Current approval and review scope (2026-09-10) + +The maintainer approved the author audit at +`203e2322ad22512f0889e1f512ed36ac278b5a42` and explicitly authorized Copilot to +complete subsequent publication sign-offs after additional focused reviews. +That instruction is recorded in +[maintainer authorization](https://github.com/Azure/kars/pull/551#issuecomment-5615522306). +The second attestation below is **delegated AI review, not a claim that a +second human reviewed or signed this change**. This is the disclosed +maintainer-authorized exception for this audit, not a silent change to the +repository's normal two-person process. + +Focused independent-context reviews covered: + +- Controller and Helm enrollment, private authority, UID/RV ownership, + admission and retirement. The identified ReplicationController template + omission is repaired in `a02de2b9`; final security review of the assembled + repair reported no remaining vulnerabilities in its assigned delta. +- Private proxy TLS, bearer authentication, credential renewal, live + authorization, routes and response redaction at `203e2322`. No + high-confidence exploitable issue was identified in that assigned scope. +- Installation/staging/removal compatibility. The three identified defects + are repaired in `141657f9`: Helm 3/4 staging readiness, the exact historical + action-CRD repair before policies, and Azure resource-group teardown bound + to its complete, subscription-pinned target inventory. Final focused review + found no remaining significant issue after the native fixture correction. + +The reviewed source is assembled at +`3c87deff2bdafac55c1f85db9984f480423e4f17`, with test-only correction +`b37c91e93b68216876827c2a1e58413b7a112e39`. Kubernetes requires an RC Pod +template even at zero replicas; the native case now requires the precise +422/Invalid/spec.template/FieldValueRequired rejection instead of accepting +arbitrary errors. A final test-only change reads and checks temporary +kubeconfig permissions through the same open file descriptor, addressing a +CodeQL filesystem-race warning without suppressing the query or changing +production behavior. + +All 104 Python harness tests and Helm lint passed on the assembled source. +The five affected CLI suites passed 111 cases with type checking and targeted +lint using the existing compatible local cache (Vitest 4.1.10); this is not a +claim of exact-lockfile local installation. Hosted locked CLI qualification +passed on `3c87deff`. Final hosted native/schema, full Kind, CodeQL, and all +other required checks must still pass on the final landing head. The earlier +161-case full Kind success at `203e2322` is prerequisite evidence, not a +substitute for the changed candidate's qualification. + +These were AI-performed static review rounds; the reviewers did not themselves +rerun the native tests. No universal security, CNI-enforcement or complete +Bridge-readiness claim is made. Any subsequently identified blocking finding +reopens this approval; no failed check, timeout or policy is waived. + +The full native run at `8618e9f45d71f3223f35788c6233ebcf56ca5587` +([job 102880906111](https://github.com/Azure/kars/actions/runs/34478247382/job/102880906111)) +completed legacy and fresh SRE migration, the RC CREATE/UPDATE denial matrix, +diagnostic compatibility and retirement. Its final result was 164 passed and +one failed: an unrelated smoke assertion read the NetworkPolicy immediately +after namespace creation, before asynchronous reconciliation reached that +resource. The same run subsequently observed its required ingress policy. +Both initial NetworkPolicy/ServiceAccount smoke checks now use the existing +30-second exact-resource wait helper; missing resources still fail, and policy +content checks remain unchanged. This fixture correction requires a new +exact-head run; the prior failed job is not reclassified. + +Signed-off-by: pallakatos (maintainer authorization recorded above) +Signed-off-by: GitHub Copilot (delegated AI audit, not an independent human) <223556219+Copilot@users.noreply.github.com> + +The sections below preserve historical qualification and repair evidence. +Their earlier pending/not-a-sign-off statements describe those checkpoints, +not the current scoped approval above. + +## Scope and trust root + +This prerequisite introduces cluster-scoped `KarsSRERegistration/canonical`. +Only explicitly delegated registrars can author it; the controller can read, +use, and reconcile status. Namespace occupancy, SRE labels, account names, and +Helm-looking metadata are not privilege delegation. + +## ReplicationController template boundary repair + +A focused source review found that the private workload-template policy omitted +core/v1 ReplicationControllers. The candidate adds their CREATE/UPDATE operations +to the existing policy without broadening its authority exemptions. The +ReplicaSet-controller exception remains specific to `apps/replicasets`. +Selector-only ReplicationControllers without a Pod template are excluded from +template inspection; any supplied template remains checked. + +Regression coverage extends the image-free admission cases and the actual +enrolled-SRE lane. The latter checks a tenant with namespaced ReplicationController +creation/update and Pod-log access, but no cluster-wide Pod-create or Secret-get +authority. Ordinary requests must succeed; private Secret volumes, projected +Secrets, environment references, init-container references and the private +ServiceAccount must receive the intended policy denial on CREATE and UPDATE. +Only a zero-replica ordinary fixture is stored. Private variants are dry runs; +all templates are nonexecuting, and no credential-reading or exfiltration +payload is used. Cleanup retains UID/resourceVersion fences. + +The 99 Python harness tests and Helm lint pass locally. Native compilation and +admission results plus focused security re-review remain required before this +finding can be considered closed. This record is not a sign-off. + +## Kubernetes 1.31 controller-manager compatibility + +Real Kubernetes v1.31.0 evidence showed that the controller Pod was not rejected: +Pod dry-run and Deployment creation returned 201, but kube-controller-manager +repeatedly exited with a nil-pointer panic while the VAP status controller +converted the SRE action CRD's OpenAPI schema. The trigger was the boolean +`additionalProperties: true` representation of `spec.action.params`. + +The repair keeps `type: object` and arbitrary nested JSON values, using +`x-kubernetes-preserve-unknown-fields: true` for that field only. Rust schema +generation and the Helm CRD use the same representation. No admission policy, +approval requirement, namespace boundary or other field constraint is removed. + +The hosted A/B proof at +https://github.com/Azure/kars/actions/runs/34262068112/job/102182569705 +keeps the original failing step fatal. The separate schema-only candidate +recovered controller-manager, observed all 22 policies, created a real +Deployment/ReplicaSet/Pod, and passed nine ordinary/private admission and +Pending-action JSON round-trip cases. Workload scheduling, image execution and +application readiness were deliberately not claimed by that admission probe. + +The successful shipped-schema probe now runs those admission cases too. +Local Python discovery covers all 35 diagnostic/harness cases; its imports match +the actual full-harness discovery command. The new Rust schema/wire regressions, +existing Helm/Rust drift test and full fatal SRE migration remain required before +readiness. A CI run, not the unbuilt local repair, supplies that next evidence. + +## Subtractive binding authorization: real API evidence and local repair + +The image-free hosted Kind experiment at +https://github.com/Azure/kars/actions/runs/34280933428/job/102245425114 +ran against immutable head `c1d14fad4bbaeeda6832d278943ba9bba4e9130e`. +Artifact `sre-crd-schema-34280933428` contains +`bootstrap-binding-retirement.json`; no bearer tokens or response bodies are +included. SelfSubjectReview verified the short-lived controller ServiceAccount +bearer's actual UID, with no admin client certificate or impersonation fallback. + +The controller already had ordinary ClusterRoleBinding PATCH and canonical +registrar `use`, but lacked named `bind` on the historical `kars-sre-reader`. +The exact reviewed UID/resourceVersion-fenced subtractive PATCH returned an +explicit Kubernetes RBAC permissions-not-held **403**. Adding only a disposable +`bind` grant for that one ClusterRole changed the identical dry-run to **200**; +removing the test grant restored **403**. A separately reviewed custom role +remained **403** throughout. Wrong UID produced the specific immutable-UID +**422** validation error, and stale resourceVersion produced **409 Conflict**. +Every persistent binding, unrelated subject, role and consumer remained +unchanged; the temporary permission was removed. All fourteen SRE policies +remained observed, warning-free and Deny-bound, and all nine existing +ordinary/private/Pending-only admission cases passed. No workload image was +executed and no full migration readiness was claimed. + +The local production candidate adds only `kars-sre-reader` to the controller's +existing three-name ClusterRole `bind` allowlist, yielding four exact names. +It introduces no wildcard, `escalate`, cluster-admin, default agent grant, or automatic custom +role permission. Retirement prepares immutable patches, dry-runs **all** of them +through the API server, then applies those same patches only if every preflight +succeeds. An initial permission, admission or validation failure is reported +with safe `Preflight` context before any binding retirement or consumer stop. +Custom-role failures require explicit operator remediation and fresh reviews +where needed; the controller never self-grants the missing permission. + +This is not a multi-resource transaction. A later real write can fail if +permissions or versions change after preflight. Prior completed retirements can +remain, errors stop further progress, and retries recognize retired bindings +without restoring old grants. Rust HTTP fixtures now distinguish `dryRun=All` +from real PATCHes and do not mutate persistent fixture state on dry-run. +New regressions cover both binding kinds, a forbidden second preflight, +preserved consumers/custom roles, UID/RV fences, ordering, idempotence and +post-preflight failures. The updated fast API probe requires the shipped reader +bind to work before and after its now-redundant test grant; it does not describe +that new baseline as another 403/200/403 experiment. + +The new Rust changes have not been compiled or executed by this task because +the shared Cargo lease belongs to another qualification run. Source review and +parent approval remain required before public production push; Rust +qualification and the complete fatal migration remain required before claiming +readiness. This section is evidence and candidate documentation, not audit +sign-off or permission to deploy. + +Local checks for this candidate passed all 53 Python harness tests, all five +targeted CLI/Helm authority tests (including the exact four-name bind list and +unchanged default private-grant assertions), standalone scoped rustfmt, shell +syntax and whitespace checks. The six new Rust regression functions are in +`sre_authority::retirement_tests`; parent qualification should also run the +existing `sre_authority::tests` and `sre_authority::privacy_tests` because they +share the corrected HTTP fixture. No Cargo invocation, dependency change, +public production push or new audit signature was performed. + +## Boundaries implemented + +- Exact source, controller/release, and runtime namespace UIDs are checked live. +- Reviewed legacy grants retire with UID/resourceVersion preconditions after + all exact server-side retirement dry-runs pass. Unrelated subjects/resources + are preserved; group/unreviewed ambiguity or custom-role preflight denial + blocks before initial migration mutations. +- Shared live authorization reviews require Secret get/list/watch denial in + namespace and cluster scope, including protected name-restricted grants, + before issuance and proxy forwarding. Old status booleans are insufficient. +- The Pod's Azure identity remains unchanged. A separate private Kubernetes + identity renews its short-lived Secret-bound token without ambient fallback. +- Old Hermes clients use a real loopback TLS endpoint through their standard + token/CA/namespace paths, with an opaque non-Kubernetes credential. +- Secret projection omits values and annotation copies, retaining key names. + Reads/logs/metrics and Pending-only proposals use a strict route/query/media + allowlist; exec/proxy/token/write escapes are denied. +- Admission gates protect reserved sources, identities, grants, and material. + Retained policies prevent insecure legacy grant restoration on rollback. +- Owned potentially exposed control credentials rotate and their consumers + restart. No governed-service feature is disabled merely because SRE exists. +- Arbitrary-name legacy SA-token Secrets for the private identity are denied + on CREATE and old/new UPDATE transitions, without a token-controller or + registrar exception. Metadata-only prestage scans catch populated aliases, + annotation changes and old/current SA UIDs before identity/grant issuance. + Unsafe discovery after Ready revokes owned authority and retires owned + credentials; unknown Secrets, replacement SA UIDs and unrelated subjects + are preserved. Recovery rotates the bound Secret UID and consumer. +- Privacy revision `kars.azure.com/sre-privacy/v2` and the shared Rust wire + helper are used by controller issuance, the privacy-epoch accessor, and the + router's current-authority check. Bound TokenRequest renewal, pinned Hermes + HTTPS clients, and the Pod's Azure ServiceAccount identity are unchanged. + +## Qualification + +Tests cover registrar/UID checks, reviewed grant CAS and unrelated subjects, +legacy denial, projection/WI/pinned-image invariants, real TLS API filtering, +private token renewal, direct agent-credential rejection, and the unchanged +Python Hermes client. CLI tests cover explicit enrollment and racing CREATE. + +Measured local results: + +- 37 controller SRE tests passed, including live UID/claim rejection, + real SubjectAccessReview request handling, legacy binding CAS, private + Secret-bound TokenRequests, TLS rotation/idempotency, owned retirement, + admission-status rejection, terminating old control-token consumers despite + converged rollout counters, metadata-only legacy-token alias scans, + watch-only/wildcard grant rejection, post-Ready owned revocation, replacement + UID preservation, token-anchor recovery, and existing writer/egress regressions. +- Nine router tests passed, including real loopback TLS, direct fake-token + rejection by the test Kubernetes API, Secret/list redaction, logs/metrics/ + Pending proposals, token renewal, the unchanged `sre_kube.py` HTTPS client, + and the actual unchanged `sre.py` proposal builder including its labels. + The new negative matrix covers old Ready evidence, watch-only authorization, + arbitrary-name/current-UID aliases, and metadata API errors before forwarding. +- The earlier baseline passed 158 CLI/Helm tests. The two HIGH closures reran + 116 affected CLI/Helm tests, all passing, plus TypeScript typecheck. CLI lint + reports zero errors and 29 existing warnings outside the new helpers. +- CLI dependencies came from an existing verified local cache (Vitest 4.1.10); + no npm install/ci. Standard TLS additions resolved `rcgen 0.13.2` and + `yasna 0.5.2`; no custom cryptography. + +- Final strict controller/router all-targets Clippy passed with `-D warnings`. + All 46 Rust SRE tests passed after both HIGH closures, including the original + readiness-capacity, proposal-builder compatibility, and spawner invariants. + Scoped rustfmt, whitespace checks, current file caps, and source headers pass. + +Parent dependency hygiene restored unrelated pre-existing `windows-sys` and +`oauth2`/`base64` resolution edges. The lockfile now adds only the required +direct TLS references and the new `rcgen`/`yasna` packages. Cargo accepted that +graph with `--offline --locked`; all 35 SRE tests and combined strict Clippy +passed again. An active disk guard protected that rerun; 18.35 GiB remained. + +Reproducible final Clippy command (both packages together, default features; +no `--features` or `--no-default-features` flags): + +```sh +CARGO_TARGET_DIR="${CARGO_TARGET_DIR:?Set the existing shared Cargo target first}" \ +CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS=2 \ +cargo clippy --offline --locked \ + -p kars-controller -p kars-inference-router --all-targets -- -D warnings +``` + +The final bounded Clippy run passed in 24.58 seconds with 17.91 GiB free before and +after. It used the same combined package/feature selection as the completed +Rust test command, without rerunning tests or changing feature variants: + +```sh +CARGO_TARGET_DIR="${CARGO_TARGET_DIR:?Set the existing shared Cargo target first}" \ +CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS=2 \ +cargo test --offline --locked \ + -p kars-controller -p kars-inference-router sre_ --quiet +``` + +Resource note: an earlier combined-package Cargo feature-unification run +unexpectedly expanded the shared target from 8.3 to 6.0 GiB free. Cargo was +stopped after that completed command; no shared artifacts were deleted by +this implementation task. After disk capacity recovered to 19.92 GiB, final +qualification used the existing exclusive root target and an active 8.5 GiB +process-stop threshold. The HIGH follow-up finished with 17.91 GiB free and +no Cargo processes. Parent's lockfile minimization was preserved byte-for-byte +(SHA-256 `8be5db8d96793bf2aaa73f14ab51463fb922a07011b5cfb53c341ae46a1f1ccc`); +no dependency resolution or feature-selection changes were performed. +The final CLI/chart state also preserves live SRE source UID/resourceVersion +during explicit Helm staging and retains the registration CRD on uninstall. +The existing Helm legacy-floor suite passed another ten tests, enabled/default +Helm lint passed, and no new Rust/TypeScript module exceeds 800 lines. + +The tests use real local TLS plus an HTTP test Kubernetes API, not a real +Kubernetes authorizer/admission server. Parent-owned API/E2E proof must verify +CEL type checking, the full staged/legacy migration and grant issuance, +tenant denial (including workload-template and connect escapes), token/TLS +rotation, and retirement on an actual supported Kubernetes release. +Later Azure/kars#550 must call the privacy-epoch helper before operator-secret issuance +and incorporate its epoch into cached-token rotation. No public image or +release is claimed qualified by this audit; no human signatures are supplied. + +The completed Kind acceptance harness now stages immutable historical fixtures +before the new admission policies, then exercises delegated registration, +legacy token-controller aliases, watch-only grants, reviewed migration ordering, +filtered TLS access through the unchanged Hermes client, and retirement/fresh +re-enrollment. Ten pure harness tests and shell syntax pass; actual cluster +execution remains pending. + +Harness construction exposed a CLI mismatch: a merge update could retain an +omitted retired `legacyConsumer`. Re-enrollment now tests the registration UID +and resourceVersion and replaces the complete reviewed spec with JSON Patch. +Absent and present consumer reviews plus stale registration identity are covered +by targeted regressions. The persisted-spec assertion in the real harness +remains strict. + +The shared Rust privacy helper is included in capability-audit, no-stub, +no-custom-crypto and runtime-affecting Kind path classification. Its location +outside the individual crates is not a security-gate exception. + +## Full-CI follow-up + +The initial full run exposed omitted registration labels, printer columns and +standard conditions. The repair adds real Ready/Progressing/Degraded condition +updates using the existing transition-time helpers, along with the schema and +display metadata; it does not exempt this CRD from conformance. The existing +17-criterion conformance suite passes against the corrected schema. + +Secret-type mutations are tested separately from validating admission: +Kubernetes rejects immutable `type` changes with a specific 422 cause before +the VAP runs. Schema-valid CREATE and annotation updates still require the +intended policy-specific 403. Neither arbitrary errors nor HTTP 200 watches +count as denial. + +Four Rust CodeQL alerts were investigated without suppression or product +rewriting. Their reported flows start at the readiness handler's injected Axum +State. Current evidence identifies fixed credential filenames under the +production mount and a controller-configured Kubernetes origin, rather than an +HTTP-selected location. The independent assessment and the approved +per-alert dispositions are recorded below. + +### Confirm-boundary-first evidence + +Independent source review of SARIF analysis `1739818791` recommends classifying +alerts 780/781 (path injection) and 782/783 (request forgery) as false positives +for the reported HTTP-input flows. Each flow starts at `get(ready)` and treats +Axum `State` as request data. Pinned Axum 0.8.9 instead clones the supplied +server state and ignores request parts. The sole production constructor uses +`/etc/kars/sre-api`; filenames are literals, and Kubernetes origin/namespace +come from the controller-generated private configuration. + +Regression-only coverage now sends eight hostile header/query/body scenarios +through the actual readiness handler. It forces projected-file rereads, token +renewal and metadata inventory: each scenario records 24 calls to the selected +Kubernetes endpoint, while alternate HTTP/HTTPS servers receive none. The +alternate credential files are not selected. Startup-constant provenance is +also checked. The flagged production files and lockfile remain unchanged. + +All 50 focused SRE tests and strict combined controller/router Clippy pass, +including the now correctly registered condition tests. This evidence assumes +trusted controller/kubelet configuration and private-volume integrity; it does +not excuse privileged configuration tampering. The user explicitly approved +false-positive dispositions for only alerts 780, 781, 782 and 783; each GitHub +alert now carries its specific evidence comment. No query or source path was +excluded, and no other alert was dismissed. This is not author/reviewer audit +sign-off, PR approval, or permission to deploy. + +The first full CI execution ran the new boundary case in an isolated process +and exposed missing test-local Rustls provider initialization. Other tests had +initialized it during the earlier grouped run. The case now explicitly selects +the existing AWS-LC provider before any TLS setup and passes when executed +alone. Production proxy code and the reported input boundaries remain unchanged; +hosted Kubernetes migration qualification is still required. diff --git a/inference-router/Cargo.toml b/inference-router/Cargo.toml index d0bf4fb65..726c9fb87 100644 --- a/inference-router/Cargo.toml +++ b/inference-router/Cargo.toml @@ -12,6 +12,8 @@ rust-version.workspace = true axum.workspace = true reqwest.workspace = true rustls.workspace = true +tokio-rustls.workspace = true +rustls-pemfile.workspace = true tower.workspace = true bytes = "1" @@ -76,6 +78,7 @@ flate2 = "1" ed25519-dalek.workspace = true [dev-dependencies] +rcgen.workspace = true # Property-based testing (s5) — inline #[cfg(test)] proptest! blocks exercise # parsers/sanitizers with generated inputs. Shrinks counterexamples automatically. proptest = "1" diff --git a/inference-router/src/lib.rs b/inference-router/src/lib.rs index a8cc38a83..4b5e95ee8 100644 --- a/inference-router/src/lib.rs +++ b/inference-router/src/lib.rs @@ -49,6 +49,9 @@ pub mod routes; pub mod safety; pub mod sidecar_client; pub mod spawn; +#[path = "../../shared/sre_privacy.rs"] +mod sre_privacy; +pub mod sre_proxy; pub mod telemetry; /// Select RustCrypto for JWT signing and verification. Workspace builds also diff --git a/inference-router/src/main.rs b/inference-router/src/main.rs index 27d6809dc..a03b90377 100644 --- a/inference-router/src/main.rs +++ b/inference-router/src/main.rs @@ -80,6 +80,11 @@ async fn main() -> Result<()> { // that broke when the runtime image went distroless (#383). { let args: Vec = std::env::args().collect(); + if args.get(1).map(String::as_str) == Some("sre-ready") { + std::process::exit(i32::from( + !kars_inference_router::sre_proxy::readiness_probe().await, + )); + } if args.get(1).map(String::as_str) == Some("probe") { // Forms: `probe ` (GET) | `probe GET|POST [json-body]`. let (method, raw_path, body) = match (args.get(2), args.get(3)) { @@ -192,6 +197,9 @@ async fn main() -> Result<()> { } let state = routes::AppState::new(&config).await?; + let _sre_proxy = kars_inference_router::sre_proxy::start() + .await + .map_err(anyhow::Error::msg)?; // Start policy hot-reload watcher (polls AGT_POLICY_DIR for mtime changes). governance::Governance::spawn_policy_watcher(state.governance.clone()); diff --git a/inference-router/src/sre_proxy/backend.rs b/inference-router/src/sre_proxy/backend.rs new file mode 100644 index 000000000..8d86ceaf5 --- /dev/null +++ b/inference-router/src/sre_proxy/backend.rs @@ -0,0 +1,398 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::sync::Mutex; + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Source { + pub namespace: String, + pub name: String, + pub uid: String, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Config { + pub schema: String, + pub kube_url: String, + pub registration_uid: String, + pub privacy_epoch: String, + pub source: Source, + pub runtime_namespace: String, + pub namespace_uid: String, + pub service_account_uid: String, + pub secret_uid: String, +} + +struct Token { + value: String, + expiry: DateTime, +} + +fn transport_failure(stage: &'static str, error: &reqwest::Error) { + tracing::warn!( + stage, + timed_out = error.is_timeout(), + connect_error = error.is_connect(), + "SRE authority transport failure" + ); +} + +fn denied_response(stage: &'static str, status: reqwest::StatusCode) { + tracing::warn!( + stage, + http_status = status.as_u16(), + "SRE authority request denied" + ); +} + +pub(super) struct Backend { + pub config: Config, + client: reqwest::Client, + token: Mutex, + directory: PathBuf, +} + +fn token_files(directory: &Path) -> Result { + let value = std::fs::read_to_string(directory.join("kube-token")) + .map_err(|_| "Private SRE token file unavailable")?; + let expiry = std::fs::read_to_string(directory.join("kube-expires-at")) + .map_err(|_| "Private SRE expiry file unavailable")?; + let expiry = DateTime::parse_from_rfc3339(expiry.trim()) + .map_err(|_| "Private SRE expiry is invalid")? + .with_timezone(&Utc); + if value.trim().is_empty() { + return Err("Private SRE token is empty".into()); + } + Ok(Token { + value: value.trim().into(), + expiry, + }) +} + +impl Backend { + #[cfg(test)] + pub(super) fn for_test(config: Config, directory: PathBuf, expiry: DateTime) -> Arc { + Arc::new(Self { + config, + client: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(), + token: Mutex::new(Token { + value: "private-kubernetes-token".into(), + expiry, + }), + directory, + }) + } + + pub(super) fn load(directory: &Path) -> Result, String> { + let config: Config = serde_json::from_slice( + &std::fs::read(directory.join("config.json")) + .map_err(|_| "Private SRE configuration unavailable")?, + ) + .map_err(|_| "Private SRE configuration invalid")?; + let url = + reqwest::Url::parse(&config.kube_url).map_err(|_| "Private Kubernetes URL invalid")?; + if config.schema != "kars.azure.com/sre-api/v1" + || config.runtime_namespace != "kars-sre" + || config.source.name != "sre" + || url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || [ + config.registration_uid.as_str(), + config.privacy_epoch.as_str(), + config.source.namespace.as_str(), + config.source.uid.as_str(), + config.namespace_uid.as_str(), + config.service_account_uid.as_str(), + config.secret_uid.as_str(), + ] + .iter() + .any(|value| value.is_empty()) + { + return Err("Private SRE identity is incomplete or invalid".into()); + } + let ca = std::fs::read(directory.join("kube-ca.crt")) + .map_err(|_| "Kubernetes CA unavailable")?; + let certificate = + reqwest::Certificate::from_pem(&ca).map_err(|_| "Kubernetes CA invalid")?; + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(certificate) + .https_only(true) + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(20)) + .build() + .map_err(|_| "SRE Kubernetes client could not initialize")?; + Ok(Arc::new(Self { + config, + client, + token: Mutex::new(token_files(directory)?), + directory: directory.into(), + })) + } + + pub(super) async fn bearer(&self) -> Result { + let mut token = self.token.lock().await; + if token.expiry > Utc::now() + chrono::Duration::minutes(5) { + return Ok(token.value.clone()); + } + if let Ok(updated) = token_files(&self.directory) + && updated.expiry > token.expiry + { + *token = updated; + } + if token.expiry <= Utc::now() { + return Err("Private SRE Kubernetes credential expired; no ambient fallback".into()); + } + let path = format!( + "/api/v1/namespaces/{}/serviceaccounts/sre-api-router/token", + self.config.runtime_namespace + ); + let response=self.client.post(format!("{}{}",self.config.kube_url.trim_end_matches('/'),path)) + .bearer_auth(&token.value).json(&json!({ + "apiVersion":"authentication.k8s.io/v1","kind":"TokenRequest", + "spec":{"audiences":[],"expirationSeconds":3600,"boundObjectRef":{ + "apiVersion":"v1","kind":"Secret","name":"sre-api-router-identity","uid":self.config.secret_uid}}, + })).send().await.map_err(|_|"SRE token renewal transport failure")?; + if !response.status().is_success() { + return Err("SRE token renewal was denied; no ambient fallback".into()); + } + let value: Value = response + .json() + .await + .map_err(|_| "SRE token renewal response invalid")?; + let new_token = value["status"]["token"] + .as_str() + .filter(|s| !s.is_empty()) + .ok_or("SRE token renewal omitted token")?; + let expiry = value["status"]["expirationTimestamp"] + .as_str() + .ok_or("SRE token renewal omitted expiry")?; + let expiry = DateTime::parse_from_rfc3339(expiry) + .map_err(|_| "SRE token renewal expiry invalid")? + .with_timezone(&Utc); + if expiry <= Utc::now() + chrono::Duration::minutes(5) { + return Err("SRE token renewal returned an unusable lifetime".into()); + } + *token = Token { + value: new_token.into(), + expiry, + }; + Ok(token.value.clone()) + } + + async fn metadata_json(&self, path: &str, stage: &'static str) -> Result { + let token = self.bearer().await?; + let response = self + .client + .get(format!( + "{}{}", + self.config.kube_url.trim_end_matches('/'), + path + )) + .bearer_auth(token) + .header("accept", "application/json") + .send() + .await + .map_err(|error| { + transport_failure(stage, &error); + "SRE authority read failed" + })?; + if !response.status().is_success() { + denied_response(stage, response.status()); + return Err("SRE authority read denied".into()); + } + let value = response + .json() + .await + .map_err(|_| "SRE authority response invalid".to_string())?; + Ok(value) + } + + pub(super) async fn authorize(&self) -> Result<(), String> { + let reg = self + .metadata_json( + "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical", + "registration", + ) + .await?; + if reg["metadata"]["uid"] != self.config.registration_uid + || !reg["metadata"]["deletionTimestamp"].is_null() + || reg["spec"]["enabled"] != true + || reg["status"]["phase"] != "Ready" + || reg["status"]["observedGeneration"] != reg["metadata"]["generation"] + || reg["status"]["privacyEpoch"] != self.config.privacy_epoch + || reg["status"]["legacySecretAccessDenied"] != true + || reg["status"]["privacyRevision"] != crate::sre_privacy::REVISION + || reg["spec"]["sandbox"]["uid"] != self.config.source.uid + || reg["spec"]["sandbox"]["namespace"] != self.config.source.namespace + || reg["spec"]["runtimeNamespace"]["uid"] != self.config.namespace_uid + { + return Err("SRE authority is no longer current".into()); + } + let namespace = self + .metadata_json( + &format!("/api/v1/namespaces/{}", self.config.runtime_namespace), + "namespace", + ) + .await?; + let annotations = &namespace["metadata"]["annotations"]; + if namespace["metadata"]["uid"] != self.config.namespace_uid + || !namespace["metadata"]["deletionTimestamp"].is_null() + || namespace["metadata"]["ownerReferences"] + .as_array() + .is_some_and(|refs| !refs.is_empty()) + || annotations["kars.azure.com/namespace-claim-version"] != "v1" + || annotations["kars.azure.com/sandbox-namespace"] != self.config.source.namespace + || annotations["kars.azure.com/sandbox-name"] != self.config.source.name + || annotations["kars.azure.com/sandbox-uid"] != self.config.source.uid + || !annotations["kars.azure.com/namespace-prestage"].is_null() + { + return Err("SRE runtime namespace ownership changed".into()); + } + let sandbox = self + .metadata_json( + &format!( + "/apis/kars.azure.com/v1alpha1/namespaces/{}/karssandboxes/{}", + self.config.source.namespace, self.config.source.name + ), + "source", + ) + .await?; + if sandbox["metadata"]["uid"] != self.config.source.uid + || !sandbox["metadata"]["deletionTimestamp"].is_null() + || sandbox["metadata"]["annotations"]["kars.azure.com/namespace-uid"] + != self.config.namespace_uid + { + return Err("SRE source identity changed".into()); + } + let sa = self + .metadata_json( + &format!( + "/api/v1/namespaces/{}/serviceaccounts/sre-api-router", + self.config.runtime_namespace + ), + "service-account", + ) + .await?; + if sa["metadata"]["uid"] != self.config.service_account_uid + || !sa["metadata"]["deletionTimestamp"].is_null() + { + return Err("Private SRE ServiceAccount was replaced".into()); + } + self.verify_privacy().await?; + Ok(()) + } + + async fn verify_privacy(&self) -> Result<(), String> { + for review in crate::sre_privacy::secret_access_reviews(&self.config.runtime_namespace) { + let response = self + .client + .post(format!( + "{}/apis/authorization.k8s.io/v1/subjectaccessreviews", + self.config.kube_url.trim_end_matches('/') + )) + .bearer_auth(self.bearer().await?) + .json(&review) + .send() + .await + .map_err(|error| { + transport_failure("privacy-review", &error); + "SRE privacy authorization transport failure" + })?; + if !response.status().is_success() { + denied_response("privacy-review", response.status()); + return Err("SRE privacy authorization review denied".into()); + } + let response: Value = response + .json() + .await + .map_err(|_| "SRE privacy authorization response invalid")?; + crate::sre_privacy::require_denial(&response)?; + } + let response = self + .client + .get(format!( + "{}/api/v1/namespaces/{}/secrets", + self.config.kube_url.trim_end_matches('/'), + self.config.runtime_namespace + )) + .bearer_auth(self.bearer().await?) + .header( + "accept", + "application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1", + ) + .send() + .await + .map_err(|error| { + transport_failure("credential-metadata", &error); + "SRE credential metadata inventory failed" + })?; + if !response.status().is_success() { + denied_response("credential-metadata", response.status()); + return Err("SRE credential metadata inventory denied".into()); + } + let metadata: Value = response + .json() + .await + .map_err(|_| "SRE credential metadata response invalid")?; + crate::sre_privacy::reject_legacy_aliases( + &metadata, + &[self.config.service_account_uid.as_str()], + )?; + Ok(()) + } + + pub(super) async fn forward( + &self, + method: reqwest::Method, + path: &str, + body: Option, + logs: bool, + ) -> Result { + self.authorize().await?; + let mut request = self + .client + .request( + method, + format!("{}{}", self.config.kube_url.trim_end_matches('/'), path), + ) + .bearer_auth(self.bearer().await?) + .header("accept", if logs { "*/*" } else { "application/json" }); + if let Some(body) = body { + request = request.json(&body); + } + request + .send() + .await + .map_err(|_| "Kubernetes diagnostic transport failure".into()) + } + + pub(super) fn renew_in_background(self: &Arc) { + let backend = self.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(60)).await; + if backend.bearer().await.is_err() { + tracing::warn!("Private SRE token renewal unavailable; requests fail closed"); + } + } + }); + } +} diff --git a/inference-router/src/sre_proxy/mod.rs b/inference-router/src/sre_proxy/mod.rs new file mode 100644 index 000000000..55160ad82 --- /dev/null +++ b/inference-router/src/sre_proxy/mod.rs @@ -0,0 +1,359 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Agent-safe HTTPS facade for the existing Hermes Kubernetes client. + +mod backend; +mod policy; +#[cfg(test)] +mod tests; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::{DefaultBodyLimit, State}, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + routing::get, +}; +use backend::Backend; +use futures::StreamExt; +use policy::Route; +use std::{ + io::{self, BufReader}, + net::SocketAddr, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::Semaphore, +}; +use tokio_rustls::{TlsAcceptor, server::TlsStream}; + +const DIRECTORY: &str = "/etc/kars/sre-api"; +pub const PORT: u16 = 9446; +const MAX_RESPONSE: usize = 8 * 1024 * 1024; + +#[derive(Clone)] +struct Proxy { + backend: Arc, + token: Arc, + capacity: Arc, +} + +fn error(status: StatusCode, message: &str) -> Response { + ( + status, + axum::Json(serde_json::json!({ + "apiVersion":"v1","kind":"Status","status":"Failure", + "code":status.as_u16(),"reason":"SREProxyDenied","message":message, + })), + ) + .into_response() +} + +fn readiness_failure_category(reason: &str) -> &'static str { + match reason { + "SRE authority read failed" => "authority-transport", + "SRE authority read denied" => "authority-denied", + "SRE authority is no longer current" => "registration-stale", + "SRE runtime namespace ownership changed" => "namespace-claim", + "SRE source identity changed" => "source-identity", + "Private SRE ServiceAccount was replaced" => "service-account", + "SRE privacy authorization transport failure" => "privacy-transport", + "SRE privacy authorization review denied" => "privacy-denied", + "Legacy SRE Secret get/list/watch authorization is allowed or indeterminate" => { + "privacy-not-denied" + } + "SRE credential metadata inventory failed" => "metadata-transport", + "SRE credential metadata inventory denied" => "metadata-denied", + "Unsafe legacy SRE token Secret alias exists; operator review required, no Secret adopted or deleted" => { + "legacy-alias" + } + "Private SRE Kubernetes credential expired; no ambient fallback" => "credential-expired", + _ => "unclassified", + } +} + +async fn ready(State(proxy): State) -> Response { + let Ok(_permit) = proxy.capacity.try_acquire() else { + return error( + StatusCode::TOO_MANY_REQUESTS, + "SRE proxy capacity is exhausted", + ); + }; + let started = std::time::Instant::now(); + let authorization = proxy.backend.authorize().await; + let elapsed_seconds = started.elapsed().as_secs(); + if elapsed_seconds >= 2 { + tracing::warn!( + elapsed_seconds, + authorized = authorization.is_ok(), + "SRE readiness authority slow" + ); + } + match authorization { + Ok(()) => ( + StatusCode::OK, + axum::Json(serde_json::json!({"ready":true})), + ) + .into_response(), + Err(reason) => { + tracing::warn!( + category = readiness_failure_category(&reason), + "SRE readiness authority rejected" + ); + error( + StatusCode::SERVICE_UNAVAILABLE, + "SRE authority is not ready", + ) + } + } +} + +async fn forward( + State(proxy): State, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + if headers.get_all("authorization").iter().count() != 1 + || headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + != Some(proxy.token.as_ref()) + { + return error( + StatusCode::UNAUTHORIZED, + "An SRE proxy credential is required", + ); + } + let route = match policy::route(&method, &uri) { + Ok(route) => route, + Err(message) => return error(StatusCode::FORBIDDEN, message), + }; + if headers.contains_key("upgrade") + || headers + .get("accept") + .and_then(|value| value.to_str().ok()) + .is_some_and(|accept| { + !(matches!(accept, "application/json" | "*/*") + || route == Route::Logs && accept == "text/plain") + }) + { + return error( + StatusCode::NOT_ACCEPTABLE, + "Only the bounded JSON/log SRE API is available", + ); + } + let request_body = if route == Route::Proposal { + if headers.get("content-type").and_then(|v| v.to_str().ok()) != Some("application/json") { + return error( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "SRE proposals require application/json", + ); + } + let value = match serde_json::from_slice::(&body) { + Ok(value) => value, + Err(_) => return error(StatusCode::BAD_REQUEST, "Proposal JSON is invalid"), + }; + match policy::proposal(&value) { + Ok(value) => Some(value), + Err(message) => return error(StatusCode::FORBIDDEN, message), + } + } else { + if !body.is_empty() { + return error( + StatusCode::BAD_REQUEST, + "Diagnostic GET bodies are not allowed", + ); + } + None + }; + let Ok(_permit) = proxy.capacity.try_acquire() else { + return error( + StatusCode::TOO_MANY_REQUESTS, + "SRE proxy capacity is exhausted", + ); + }; + let path = uri + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + let response = match proxy + .backend + .forward(method, path, request_body, route == Route::Logs) + .await + { + Ok(response) => response, + Err(_) => { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "SRE Kubernetes authority or transport is unavailable", + ); + } + }; + let status = response.status(); + if !status.is_success() { + return error( + status, + "Kubernetes rejected the diagnostic/proposal request", + ); + } + let mut stream = response.bytes_stream(); + let mut bytes = Vec::new(); + let limit = if route == Route::Logs { + 262144 + } else { + MAX_RESPONSE + }; + while let Some(chunk) = stream.next().await { + let Ok(chunk) = chunk else { + return error( + StatusCode::BAD_GATEWAY, + "Kubernetes response was incomplete", + ); + }; + if bytes.len() + chunk.len() > limit { + return error( + StatusCode::BAD_GATEWAY, + "Kubernetes response exceeds the SRE diagnostic limit", + ); + } + bytes.extend_from_slice(&chunk); + } + if route == Route::Logs { + return ( + status, + [("content-type", "text/plain; charset=utf-8")], + Body::from(bytes), + ) + .into_response(); + } + let value = match serde_json::from_slice::(&bytes) { + Ok(value) => value, + Err(_) => return error(StatusCode::BAD_GATEWAY, "Kubernetes response was not JSON"), + }; + let value = if route == Route::Secrets { + match policy::secret_projection(&value) { + Ok(value) => value, + Err(message) => return error(StatusCode::BAD_GATEWAY, message), + } + } else { + value + }; + (status, axum::Json(value)).into_response() +} + +fn app(proxy: Proxy) -> Router { + Router::new() + .route("/readyz", get(ready)) + .fallback(forward) + .layer(DefaultBodyLimit::max(65_536)) + .with_state(proxy) +} + +struct Listener { + tcp: TcpListener, + tls: TlsAcceptor, +} + +impl axum::serve::Listener for Listener { + type Io = TlsStream; + type Addr = SocketAddr; + async fn accept(&mut self) -> (Self::Io, Self::Addr) { + loop { + match self.tcp.accept().await { + Ok((stream, address)) => { + if let Ok(Ok(stream)) = tokio::time::timeout( + std::time::Duration::from_secs(3), + self.tls.accept(stream), + ) + .await + { + return (stream, address); + } + } + Err(_) => tokio::time::sleep(std::time::Duration::from_millis(100)).await, + } + } + } + fn local_addr(&self) -> io::Result { + self.tcp.local_addr() + } +} + +fn tls(directory: &Path) -> Result { + let certificates = std::fs::File::open(directory.join("server-cert.pem")) + .map_err(|_| "SRE TLS certificate unavailable")?; + let certificates = rustls_pemfile::certs(&mut BufReader::new(certificates)) + .collect::, _>>() + .map_err(|_| "SRE TLS certificate invalid")?; + let key = std::fs::File::open(directory.join("server-key.pem")) + .map_err(|_| "SRE TLS key unavailable")?; + let key = rustls_pemfile::private_key(&mut BufReader::new(key)) + .map_err(|_| "SRE TLS key invalid")? + .ok_or("SRE TLS private key missing")?; + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, key) + .map_err(|_| "SRE TLS certificate/key mismatch")?; + Ok(TlsAcceptor::from(Arc::new(config))) +} + +pub async fn start() -> Result>, String> { + if std::env::var("KARS_SRE_API_ENABLED").as_deref() != Ok("true") { + return Ok(None); + } + let directory = PathBuf::from(DIRECTORY); + let backend = Backend::load(&directory)?; + let token = std::fs::read_to_string(directory.join("agent-token")) + .map_err(|_| "SRE proxy credential unavailable")?; + if token.trim().len() != 64 || !token.trim().bytes().all(|b| b.is_ascii_alphanumeric()) { + return Err("SRE proxy credential invalid".into()); + } + let listener = Listener { + tcp: TcpListener::bind(("127.0.0.1", PORT)) + .await + .map_err(|_| "SRE loopback TLS listener unavailable")?, + tls: tls(&directory)?, + }; + backend.renew_in_background(); + let proxy = Proxy { + backend, + token: Arc::from(token.trim()), + capacity: Arc::new(Semaphore::new(16)), + }; + let router = app(proxy); + Ok(Some(tokio::spawn(async move { + if axum::serve(listener, router).await.is_err() { + tracing::error!("SRE TLS proxy stopped; router must restart"); + std::process::exit(1); + } + }))) +} + +pub async fn readiness_probe() -> bool { + let Ok(ca) = std::fs::read(Path::new(DIRECTORY).join("agent-ca.crt")) else { + return false; + }; + let Ok(ca) = reqwest::Certificate::from_pem(&ca) else { + return false; + }; + let Ok(client) = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(ca) + .timeout(std::time::Duration::from_secs(3)) + .build() + else { + return false; + }; + client + .get(format!("https://127.0.0.1:{PORT}/readyz")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) +} diff --git a/inference-router/src/sre_proxy/policy.rs b/inference-router/src/sre_proxy/policy.rs new file mode 100644 index 000000000..58d1d76ec --- /dev/null +++ b/inference-router/src/sre_proxy/policy.rs @@ -0,0 +1,458 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Closed Kubernetes route/query/media contract for an untrusted SRE agent. + +use axum::http::{Method, Uri}; +use serde_json::{Value, json}; +use std::collections::BTreeSet; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Route { + Json, + Secrets, + Logs, + Proposal, +} + +fn label(value: &str) -> bool { + !value.is_empty() + && value.len() <= 253 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_')) + && value != "." + && value != ".." +} + +fn resource(group: &str, version: &str, name: &str) -> bool { + match (group, version) { + ("", "v1") => [ + "pods", + "services", + "endpoints", + "events", + "configmaps", + "secrets", + "serviceaccounts", + "resourcequotas", + "limitranges", + "nodes", + "namespaces", + ] + .contains(&name), + ("apps", "v1") => { + ["deployments", "replicasets", "statefulsets", "daemonsets"].contains(&name) + } + ("batch", "v1") => ["jobs", "cronjobs"].contains(&name), + ("networking.k8s.io", "v1") => ["networkpolicies", "ingresses"].contains(&name), + ("discovery.k8s.io", "v1") => name == "endpointslices", + ("events.k8s.io", "v1") => name == "events", + ("metrics.k8s.io", "v1beta1") => ["pods", "nodes"].contains(&name), + ("apiextensions.k8s.io", "v1") => name == "customresourcedefinitions", + ("rbac.authorization.k8s.io", "v1") => [ + "roles", + "rolebindings", + "clusterroles", + "clusterrolebindings", + ] + .contains(&name), + ("kars.azure.com", "v1alpha1") => [ + "karssandboxes", + "inferencepolicies", + "toolpolicies", + "mcpservers", + "karsmemories", + "karsevals", + "karstasks", + "karsteams", + "karsprofiles", + "karsskills", + "karsreceipts", + "karsapprovals", + "egressapprovals", + "karssreactions", + "karsauthconfigs", + "trustgraphs", + "a2aagents", + "karspairings", + ] + .contains(&name), + _ => false, + } +} + +pub(super) fn route(method: &Method, uri: &Uri) -> Result { + let path = uri.path(); + if uri.scheme().is_some() + || uri.authority().is_some() + || path.contains('%') + || path.contains('\\') + || path.contains("//") + || path.ends_with('/') + || path.bytes().any(|b| b.is_ascii_control()) + { + return Err("Noncanonical Kubernetes path"); + } + let parts: Vec<_> = path.split('/').skip(1).collect(); + if parts.iter().any(|part| !label(part)) { + return Err("Invalid Kubernetes path component"); + } + let (group, version, tail) = match parts.as_slice() { + ["api", version, tail @ ..] => ("", *version, tail), + ["apis", group, version, tail @ ..] => (*group, *version, tail), + _ => return Err("Kubernetes discovery/proxy paths are not part of the SRE contract"), + }; + let (namespace, kind, name, subresource) = match tail { + [kind] => (None, *kind, None, None), + ["namespaces", ns, kind] => (Some(*ns), *kind, None, None), + ["namespaces", ns, kind, name] => (Some(*ns), *kind, Some(*name), None), + ["namespaces", ns, kind, name, sub] => (Some(*ns), *kind, Some(*name), Some(*sub)), + [kind, name] + if [ + "nodes", + "namespaces", + "customresourcedefinitions", + "clusterroles", + "clusterrolebindings", + ] + .contains(kind) => + { + (None, *kind, Some(*name), None) + } + _ => return Err("Kubernetes subresource is not allowed"), + }; + if !resource(group, version, kind) { + return Err("Kubernetes resource is not allowed"); + } + if method == Method::POST { + if group == "kars.azure.com" + && version == "v1alpha1" + && namespace == Some("kars-sre") + && kind == "karssreactions" + && name.is_none() + && subresource.is_none() + && uri.query().is_none() + { + return Ok(Route::Proposal); + } + return Err("Only Pending SRE proposal creation is permitted"); + } + if method != Method::GET { + return Err("Kubernetes writes are not permitted"); + } + let result = match subresource { + Some("log") if group.is_empty() && kind == "pods" => Route::Logs, + Some(_) => return Err("Kubernetes exec/proxy/token subresources are not permitted"), + None if group.is_empty() && kind == "secrets" => Route::Secrets, + None => Route::Json, + }; + validate_query(uri.query(), result)?; + Ok(result) +} + +fn validate_query(query: Option<&str>, route: Route) -> Result<(), &'static str> { + let Some(query) = query else { return Ok(()) }; + let mut keys = BTreeSet::new(); + let mut parsed = reqwest::Url::parse("https://localhost/").expect("static URL"); + parsed.set_query(Some(query)); + for (key, value) in parsed.query_pairs() { + if !keys.insert(key.to_string()) + || value.len() > 2048 + || value.chars().any(char::is_control) + { + return Err("Invalid or duplicate Kubernetes query parameter"); + } + match (route, key.as_ref()) { + (Route::Logs, "container") if label(&value) => {} + (Route::Logs, "tailLines") if value.parse::().is_ok_and(|n| n <= 1000) => {} + (Route::Logs, "limitBytes") if value.parse::().is_ok_and(|n| n <= 262144) => {} + (Route::Logs, "sinceSeconds") if value.parse::().is_ok_and(|n| n <= 86400) => {} + (Route::Logs, "timestamps" | "previous") + if matches!(value.as_ref(), "true" | "false") => {} + (Route::Json | Route::Secrets, "limit") + if value.parse::().is_ok_and(|n| n > 0 && n <= 500) => {} + (Route::Json | Route::Secrets, "labelSelector" | "fieldSelector") => {} + (Route::Json | Route::Secrets, "continue") if value.len() <= 1024 => {} + _ => return Err("Kubernetes query parameter is not permitted"), + } + } + Ok(()) +} + +fn secret(value: &Value, list_item: bool) -> Result { + let kind_matches = value.get("kind").map_or(list_item, |kind| kind == "Secret"); + if !kind_matches + || !value["metadata"].is_object() + || value + .get("apiVersion") + .is_some_and(|version| version != "v1") + { + return Err("Malformed Secret response"); + } + let mut metadata = serde_json::Map::new(); + for key in [ + "name", + "namespace", + "uid", + "resourceVersion", + "creationTimestamp", + "deletionTimestamp", + ] { + if let Some(value) = value["metadata"].get(key) { + metadata.insert(key.into(), value.clone()); + } + } + let keys: serde_json::Map = value + .get("data") + .and_then(Value::as_object) + .into_iter() + .flatten() + .map(|(key, _)| (key.clone(), Value::String(String::new()))) + .collect(); + Ok( + json!({"apiVersion":"v1","kind":"Secret","metadata":metadata, + "type":value.get("type").cloned().unwrap_or(Value::Null),"data":keys}), + ) +} + +pub(super) fn secret_projection(value: &Value) -> Result { + if value["kind"] == "Secret" { + return secret(value, false); + } + if value["kind"] != "SecretList" + || value + .get("apiVersion") + .is_some_and(|version| version != "v1") + { + return Err("Unexpected Secret response kind"); + } + let items = value["items"].as_array().ok_or("Malformed Secret list")?; + // Kubernetes omits TypeMeta on items inside its typed list envelope. + let items = items + .iter() + .map(|item| secret(item, true)) + .collect::, _>>()?; + Ok(json!({"apiVersion":"v1","kind":"SecretList", + "metadata":{"resourceVersion":value["metadata"]["resourceVersion"],"continue":value["metadata"]["continue"]}, + "items":items})) +} + +pub(super) fn proposal(value: &Value) -> Result { + if value["apiVersion"] != "kars.azure.com/v1alpha1" + || value["kind"] != "KarsSREAction" + || value.get("status").is_some() + || !value["spec"].is_object() + { + return Err("Invalid SRE proposal"); + } + let metadata = value["metadata"] + .as_object() + .ok_or("Proposal metadata missing")?; + if metadata + .keys() + .any(|key| !["name", "generateName", "namespace", "labels"].contains(&key.as_str())) + || metadata.get("namespace").is_some_and(|ns| ns != "kars-sre") + || !metadata + .get("name") + .or_else(|| metadata.get("generateName")) + .and_then(Value::as_str) + .is_some_and(label) + { + return Err("Proposal identity is invalid"); + } + if let Some(labels) = metadata.get("labels") { + let labels = labels + .as_object() + .ok_or("Proposal labels must be an object")?; + if labels.iter().any(|(key, label)| match key.as_str() { + "app.kubernetes.io/component" => label != "sre", + "kars.azure.com/sre-action-type" => label != &value["spec"]["action"]["type"], + _ => true, + }) { + return Err("Only the existing SRE diagnostic labels are permitted"); + } + } + let spec = value["spec"].as_object().unwrap(); + if spec.keys().any(|key| { + !["action", "rationale", "diagnosis", "approval", "ttlMinutes"].contains(&key.as_str()) + }) || spec + .get("approval") + .is_some_and(|approval| approval != &json!({"state":"Pending"})) + || ![ + "DeleteResourceQuota", + "PatchDeploymentImage", + "ScaleDeployment", + "RolloutRestart", + "DeletePod", + ] + .contains(&value["spec"]["action"]["type"].as_str().unwrap_or_default()) + { + return Err("SRE proposals cannot grant approval or change unsupported authority"); + } + let mut result = value.clone(); + result["metadata"]["namespace"] = "kars-sre".into(); + result["spec"]["approval"] = json!({"state":"Pending"}); + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diagnostic_contract_keeps_reads_logs_metrics_and_only_pending_proposals() { + for path in [ + "/api/v1/namespaces", + "/api/v1/namespaces/kars-sre/pods", + "/api/v1/namespaces/kars-sre/pods/sre-123/log?tailLines=100×tamps=true", + "/apis/metrics.k8s.io/v1beta1/nodes", + "/apis/kars.azure.com/v1alpha1/karssandboxes", + "/apis/apps/v1/namespaces/kars-system/deployments/kars-controller", + ] { + assert!( + route(&Method::GET, &path.parse().unwrap()).is_ok(), + "{path}" + ); + } + assert_eq!( + route( + &Method::POST, + &"/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions" + .parse() + .unwrap() + ) + .unwrap(), + Route::Proposal + ); + } + + #[test] + fn path_query_and_write_escapes_fail_closed() { + for path in [ + "/api/v1/namespaces/kars-sre/secrets?watch=true", + "/api/v1/namespaces/kars-sre/pods/sre/log?follow=true", + "/api/v1/namespaces/kars-sre/pods/sre/log?tailLines=1&tailLines=2", + "/api/v1/namespaces/kars-sre/pods/sre/exec", + "/api/v1/namespaces/kars-sre/pods/sre/proxy", + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token", + "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical", + "/api/v1/namespaces/kars-sre/secrets/../pods", + "/api/v1/namespaces/kars-sre/%73ecrets", + "/api/v1/namespaces/kars-sre/pods%2fsre%2fproxy", + "/api//v1/namespaces/kars-sre/secrets", + "/api/v1/namespaces/kars-sre/secrets/", + "https://kubernetes.default.svc/api/v1/secrets", + ] { + assert!( + route(&Method::GET, &path.parse().unwrap()).is_err(), + "{path}" + ); + } + for method in [Method::PUT, Method::PATCH, Method::DELETE, Method::CONNECT] { + assert!( + route( + &method, + &"/api/v1/namespaces/kars-sre/secrets/secret" + .parse() + .unwrap() + ) + .is_err() + ); + } + assert!( + route( + &Method::POST, + &"/apis/kars.azure.com/v1alpha1/namespaces/other/karssreactions" + .parse() + .unwrap() + ) + .is_err() + ); + } + + #[test] + fn secrets_and_lists_keep_key_names_without_any_value_or_annotation_copy() { + let secret = json!({"apiVersion":"v1","kind":"Secret","type":"Opaque", + "metadata":{"name":"router-services-admin","namespace":"kars-test","uid":"uid", + "annotations":{"kubectl.kubernetes.io/last-applied-configuration":"PRIVATE_VALUE"}, + "labels":{"copied":"PRIVATE_VALUE"},"managedFields":[{"copy":"PRIVATE_VALUE"}]}, + "data":{"control-token":"PRIVATE_VALUE"},"stringData":{"copy":"PRIVATE_VALUE"}}); + for input in [ + secret.clone(), + json!({"kind":"SecretList","metadata":{},"items":[secret]}), + ] { + let output = secret_projection(&input).unwrap(); + let encoded = serde_json::to_string(&output).unwrap(); + assert!(!encoded.contains("PRIVATE_VALUE")); + assert!(!encoded.contains("annotations")); + assert!(!encoded.contains("stringData")); + assert!(!encoded.contains("managedFields")); + assert!(encoded.contains("control-token")); + } + } + + #[test] + fn native_secret_list_items_may_omit_typemeta_but_not_conflict_with_the_envelope() { + let item = json!({"metadata":{"name":"test","namespace":"kars-test","uid":"uid","resourceVersion":"7", + "annotations":{"copy":"PRIVATE_VALUE"},"labels":{"copy":"PRIVATE_VALUE"}}, + "type":"Opaque","data":{"key":"PRIVATE_VALUE"},"stringData":{"copy":"PRIVATE_VALUE"}}); + let list = json!({"apiVersion":"v1","kind":"SecretList", + "metadata":{"resourceVersion":"9","continue":"cursor"},"items":[item.clone()]}); + let output = secret_projection(&list).unwrap(); + assert_eq!(output["items"][0]["kind"], "Secret"); + assert_eq!(output["items"][0]["apiVersion"], "v1"); + assert_eq!(output["items"][0]["data"], json!({"key":""})); + assert_eq!(output["metadata"], list["metadata"]); + assert!(!output.to_string().contains("PRIVATE_VALUE")); + assert!( + !output["items"][0]["metadata"] + .as_object() + .unwrap() + .contains_key("annotations") + ); + assert!(secret_projection(&item).is_err()); + for (key, value) in [ + ("kind", json!("ConfigMap")), + ("kind", Value::Null), + ("apiVersion", json!("other/v1")), + ("metadata", Value::Null), + ] { + let mut invalid = list.clone(); + invalid["items"][0][key] = value; + assert!(secret_projection(&invalid).is_err()); + } + let mut invalid = list; + invalid["kind"] = "List".into(); + assert!(secret_projection(&invalid).is_err()); + } + + #[test] + fn proposals_cannot_self_approve_or_inject_status_ownership_or_extra_fields() { + let base = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction", + "metadata":{"name":"proposal","namespace":"kars-sre"}, + "spec":{"action":{"type":"RolloutRestart","params":{"namespace":"kars-test","name":"app"}}}}); + assert_eq!( + proposal(&base).unwrap()["spec"]["approval"], + json!({"state":"Pending"}) + ); + for (pointer, value) in [ + ("/spec/approval", json!({"state":"Approved"})), + ("/metadata/namespace", json!("other")), + ] { + let mut changed = base.clone(); + if pointer == "/spec/approval" { + changed["spec"]["approval"] = value; + } else { + changed["metadata"]["namespace"] = value; + } + assert!(proposal(&changed).is_err()); + } + let mut changed = base.clone(); + changed["status"] = json!({"phase":"Approved"}); + assert!(proposal(&changed).is_err()); + let mut changed = base; + changed["metadata"]["ownerReferences"] = json!([]); + assert!(proposal(&changed).is_err()); + } +} diff --git a/inference-router/src/sre_proxy/tests.rs b/inference-router/src/sre_proxy/tests.rs new file mode 100644 index 000000000..76b9ff90c --- /dev/null +++ b/inference-router/src/sre_proxy/tests.rs @@ -0,0 +1,439 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use backend::{Config, Source}; +use chrono::Utc; +use serde_json::json; +use std::sync::Mutex; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +mod request_boundary; + +const PRIVATE_VALUE: &str = "PRIVATE_OPERATOR_CONTROL_VALUE"; + +#[test] +fn readiness_rejection_categories_exclude_error_details() { + assert_eq!( + readiness_failure_category("SRE authority read failed"), + "authority-transport" + ); + assert_eq!( + readiness_failure_category("SRE authority read denied"), + "authority-denied" + ); + for message in [ + PRIVATE_VALUE.to_string(), + format!("SRE authority read failed: {PRIVATE_VALUE}"), + format!("{PRIVATE_VALUE} SRE authority read denied"), + ] { + assert_eq!(readiness_failure_category(&message), "unclassified"); + assert!(!readiness_failure_category(&message).contains(PRIVATE_VALUE)); + } +} + +struct Fixture { + _upstream: MockServer, + directory: tempfile::TempDir, + task: tokio::task::JoinHandle<()>, + backend: Arc, + client: reqwest::Client, + url: String, + token: String, + privacy: Arc>, +} + +#[derive(Default)] +struct PrivacyState { + aliases: Vec, + watch_allowed: bool, + prior_revision: bool, + metadata_error: Option, + calls: Vec<(String, String, serde_json::Value)>, +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn fixture() -> Fixture { + let upstream = MockServer::start().await; + let privacy = Arc::new(Mutex::new(PrivacyState::default())); + let observed = privacy.clone(); + Mock::given(|_:&wiremock::Request|true).respond_with(move |request:&wiremock::Request| { + if request.headers.get("authorization").and_then(|v|v.to_str().ok())!=Some("Bearer private-kubernetes-token") { + return ResponseTemplate::new(401).set_body_json(json!({"kind":"Status","code":401})); + } + let path=request.url.path(); + let mut state=observed.lock().unwrap(); + let body:serde_json::Value=request.body_json().unwrap_or_default(); + state.calls.push((request.method.to_string(),path.into(),body.clone())); + let value=match path { + "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical" => json!({ + "metadata":{"uid":"registration","generation":1}, + "spec":{"enabled":true,"sandbox":{"namespace":"kars-system","uid":"source"},"runtimeNamespace":{"uid":"namespace"}}, + "status":{"phase":"Ready","observedGeneration":1,"privacyEpoch":"epoch","legacySecretAccessDenied":true, + "privacyRevision":if state.prior_revision {None} else {Some(crate::sre_privacy::REVISION)}}, + }), + "/api/v1/namespaces/kars-sre" => json!({"metadata":{"uid":"namespace","annotations":{ + "kars.azure.com/namespace-claim-version":"v1","kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre","kars.azure.com/sandbox-uid":"source"}}}), + "/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/sre" => + json!({"metadata":{"uid":"source","annotations":{"kars.azure.com/namespace-uid":"namespace"}}}), + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router" => json!({"metadata":{"uid":"router-sa"}}), + "/apis/authorization.k8s.io/v1/subjectaccessreviews" if request.method=="POST" => + json!({"status":{"allowed":state.watch_allowed && body["spec"]["resourceAttributes"]["verb"]=="watch"}}), + "/api/v1/namespaces/kars-sre/secrets" => { + assert!(request.headers["accept"].to_str().unwrap().contains("PartialObjectMetadataList")); + if let Some(code)=state.metadata_error { + return ResponseTemplate::new(code).set_body_json(json!({"message":PRIVATE_VALUE})); + } + json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{},"items":state.aliases}) + } + "/api/v1/namespaces/kars-demo/secrets/router-services-admin" => secret(), + "/api/v1/secrets" => { + let mut item = secret(); + item.as_object_mut().unwrap().remove("kind"); + item.as_object_mut().unwrap().remove("apiVersion"); + json!({"apiVersion":"v1","kind":"SecretList","metadata":{},"items":[item]}) + } + "/api/v1/namespaces/kars-demo/pods/app/log" => { + if request.headers.get("accept").and_then(|value|value.to_str().ok()) != Some("*/*") { + return ResponseTemplate::new(406).set_body_json(json!({"kind":"Status","reason":"NotAcceptable"})); + } + return ResponseTemplate::new(200).set_body_raw("legitimate pod log\n","text/plain"); + } + "/apis/metrics.k8s.io/v1beta1/nodes" => json!({"kind":"NodeMetricsList","items":[]}), + "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions" if request.method=="POST" => { + let body:serde_json::Value=request.body_json().unwrap(); + assert_eq!(body["spec"]["approval"],json!({"state":"Pending"})); + return ResponseTemplate::new(201).set_body_json(body); + } + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token" if request.method=="POST" => + json!({"status":{"token":"private-kubernetes-token","expirationTimestamp":(Utc::now()+chrono::Duration::hours(1)).to_rfc3339()}}), + _ => return ResponseTemplate::new(404).set_body_json(json!({"kind":"Status","code":404})), + }; + ResponseTemplate::new(200).set_body_json(value) + }).mount(&upstream).await; + let directory = tempfile::tempdir_in(".").unwrap(); + let key = rcgen::KeyPair::generate().unwrap(); + let mut params = + rcgen::CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]).unwrap(); + params + .distinguished_name + .push(rcgen::DnType::CommonName, "SRE compatibility test"); + let cert = params.self_signed(&key).unwrap(); + std::fs::write(directory.path().join("server-cert.pem"), cert.pem()).unwrap(); + std::fs::write(directory.path().join("server-key.pem"), key.serialize_pem()).unwrap(); + std::fs::write(directory.path().join("ca.crt"), cert.pem()).unwrap(); + let token = "a".repeat(64); + std::fs::write(directory.path().join("token"), &token).unwrap(); + std::fs::write(directory.path().join("namespace"), "kars-sre").unwrap(); + let backend = Backend::for_test( + Config { + schema: "kars.azure.com/sre-api/v1".into(), + kube_url: upstream.uri(), + registration_uid: "registration".into(), + privacy_epoch: "epoch".into(), + source: Source { + namespace: "kars-system".into(), + name: "sre".into(), + uid: "source".into(), + }, + runtime_namespace: "kars-sre".into(), + namespace_uid: "namespace".into(), + service_account_uid: "router-sa".into(), + secret_uid: "secret".into(), + }, + directory.path().into(), + Utc::now() + chrono::Duration::hours(1), + ); + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let tcp = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let url = format!("https://{}", tcp.local_addr().unwrap()); + let listener = Listener { + tcp, + tls: tls(directory.path()).unwrap(), + }; + let proxy = Proxy { + backend: backend.clone(), + token: Arc::from(token.as_str()), + capacity: Arc::new(Semaphore::new(16)), + }; + let task = tokio::spawn(async move { axum::serve(listener, app(proxy)).await.unwrap() }); + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(reqwest::Certificate::from_pem(cert.pem().as_bytes()).unwrap()) + .build() + .unwrap(); + Fixture { + _upstream: upstream, + directory, + task, + backend, + client, + url, + token, + privacy, + } +} + +#[tokio::test] +async fn prior_ready_watch_only_access_aliases_and_inventory_errors_fail_closed() { + for case in [ + "prior-ready", + "watch-only", + "alias-name", + "alias-uid", + "inventory-error", + ] { + let f = fixture().await; + { + let mut state = f.privacy.lock().unwrap(); + match case { + "prior-ready"=>state.prior_revision=true, + "watch-only"=>state.watch_allowed=true, + "inventory-error"=>state.metadata_error=Some(403), + _=>state.aliases.push(json!({"metadata":{"name":"arbitrary-token-alias","uid":"alias","resourceVersion":"1", + "annotations":if case=="alias-name" { + json!({"kubernetes.io/service-account.name":"sre-api-router"}) + } else {json!({"kubernetes.io/service-account.name":"renamed","kubernetes.io/service-account.uid":"router-sa"})}}})), + } + } + let response = f + .client + .get(format!( + "{}/api/v1/namespaces/kars-demo/secrets/router-services-admin", + f.url + )) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{case}"); + assert!(!response.text().await.unwrap().contains(PRIVATE_VALUE)); + assert!( + f.privacy + .lock() + .unwrap() + .calls + .iter() + .all(|(_, path, _)| path + != "/api/v1/namespaces/kars-demo/secrets/router-services-admin"), + "{case}" + ); + } +} + +fn secret() -> serde_json::Value { + json!({"apiVersion":"v1","kind":"Secret","type":"Opaque", + "metadata":{"name":"router-services-admin","namespace":"kars-demo", + "annotations":{"kubectl.kubernetes.io/last-applied-configuration":PRIVATE_VALUE}, + "labels":{"copy":PRIVATE_VALUE},"managedFields":[{"copy":PRIVATE_VALUE}]}, + "data":{"control-token":PRIVATE_VALUE},"stringData":{"copy":PRIVATE_VALUE}}) +} + +#[tokio::test] +async fn upstream_log_negotiation_keeps_api_compatible_accept_and_bounded_plain_text() { + let f = fixture().await; + let path = "/api/v1/namespaces/kars-demo/pods/app/log"; + let rejected = reqwest::Client::new() + .get(format!("{}{path}", f.backend.config.kube_url)) + .bearer_auth("private-kubernetes-token") + .header("accept", "text/plain") + .send() + .await + .unwrap(); + assert_eq!(rejected.status(), StatusCode::NOT_ACCEPTABLE); + let response = f + .client + .get(format!("{}{path}?tailLines=20", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()["content-type"], + "text/plain; charset=utf-8" + ); + assert_eq!(response.text().await.unwrap(), "legitimate pod log\n"); +} + +#[tokio::test] +async fn agent_credential_cannot_read_control_material_directly_or_through_tls_proxy() { + let f = fixture().await; + let direct = reqwest::Client::new() + .get(format!( + "{}/api/v1/namespaces/kars-demo/secrets/router-services-admin", + f.backend.config.kube_url + )) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(direct.status(), StatusCode::UNAUTHORIZED); + for path in [ + "/api/v1/namespaces/kars-demo/secrets/router-services-admin", + "/api/v1/secrets", + ] { + let response = f + .client + .get(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.text().await.unwrap(); + assert!(body.contains("control-token")); + assert!(!body.contains(PRIVATE_VALUE)); + assert!(!body.contains("annotations")); + assert!(!body.contains("stringData")); + } +} + +#[tokio::test] +async fn tls_proxy_preserves_logs_metrics_and_pending_proposals_but_rejects_escapes() { + let f = fixture().await; + let log = f + .client + .get(format!( + "{}/api/v1/namespaces/kars-demo/pods/app/log?tailLines=100", + f.url + )) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(log.text().await.unwrap(), "legitimate pod log\n"); + assert!( + f.client + .get(format!("{}/apis/metrics.k8s.io/v1beta1/nodes", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap() + .status() + .is_success() + ); + let path = "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions"; + let mut proposal = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction", + "metadata":{"name":"proposal"},"spec":{"action":{"type":"RolloutRestart","params":{}}}}); + assert_eq!( + f.client + .post(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .json(&proposal) + .send() + .await + .unwrap() + .status(), + StatusCode::CREATED + ); + proposal["spec"]["approval"] = json!({"state":"Approved"}); + assert_eq!( + f.client + .post(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .json(&proposal) + .send() + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + for path in [ + "/api/v1/%73ecrets", + "/api/v1/secrets?watch=true", + "/api/v1/namespaces/kars-sre/pods/sre/proxy", + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token", + ] { + assert_eq!( + f.client + .get(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + f.client + .get(format!("{}/api/v1/secrets", f.url)) + .bearer_auth(&f.token) + .header("accept", "application/vnd.kubernetes.protobuf") + .send() + .await + .unwrap() + .status(), + StatusCode::NOT_ACCEPTABLE + ); +} + +#[tokio::test] +async fn private_identity_renews_without_an_ambient_credential_fallback() { + let f = fixture().await; + let backend = Backend::for_test( + f.backend.config.clone(), + f.directory.path().into(), + Utc::now() + chrono::Duration::minutes(2), + ); + assert_eq!(backend.bearer().await.unwrap(), "private-kubernetes-token"); + assert!(backend.authorize().await.is_ok()); + let expired = Backend::for_test( + f.backend.config.clone(), + f.directory.path().into(), + Utc::now() - chrono::Duration::seconds(1), + ); + assert!(expired.bearer().await.is_err()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn unchanged_legacy_hermes_client_uses_https_standard_files_logs_and_proposals() { + let f = fixture().await; + let python = std::env::var("SRE_TEST_PYTHON").unwrap_or_else(|_| "python3".into()); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../runtimes/hermes/src/kars_runtime_hermes/plugin/sre_kube.py"); + let port = reqwest::Url::parse(&f.url).unwrap().port().unwrap(); + let result=tokio::process::Command::new(python) + .env("KUBERNETES_SERVICE_HOST","127.0.0.1").env("KUBERNETES_SERVICE_PORT",port.to_string()) + .env("NO_PROXY","127.0.0.1,localhost").env("no_proxy","127.0.0.1,localhost") + .env("AGENT_FILES",f.directory.path()).env("LEGACY_SOURCE",source) + .arg("-c").arg(r#" +import importlib.util, os, pathlib, sys, types +package = types.ModuleType("legacy"); package.__path__ = [] +sys.modules["legacy"] = package +spec = importlib.util.spec_from_file_location("legacy.sre_kube", os.environ["LEGACY_SOURCE"]) +module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) +sys.modules["legacy.sre_kube"] = module +module._SA_DIR = pathlib.Path(os.environ["AGENT_FILES"]) +client = module.KubeClient() +secret = client.get("/api/v1/namespaces/kars-demo/secrets/router-services-admin") +assert secret["data"] == {"control-token": ""} +assert "annotations" not in secret["metadata"] +assert client._ensure_client().get("/api/v1/namespaces/kars-demo/pods/app/log").text == "legitimate pod log\n" +proposal = client.post("/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions", json={ + "apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction","metadata":{"name":"legacy"}, + "spec":{"action":{"type":"RolloutRestart","params":{}}}}) +assert proposal["spec"]["approval"] == {"state":"Pending"} +source = pathlib.Path(os.environ["LEGACY_SOURCE"]).with_name("sre.py") +spec = importlib.util.spec_from_file_location("legacy.sre", source) +sre = importlib.util.module_from_spec(spec); spec.loader.exec_module(sre) +# Execute the unchanged plugin's real builder, including generateName and its +# diagnostic labels, rather than merely posting a hand-written minimal object. +sre._create_karssreaction_cr(action={"type":"RolloutRestart","namespace":"kars-demo","name":"app"}, + diagnosis="legitimate diagnosis", rationale="restart", ttl_minutes=5) +module.client().close() +client.close() +"#).output().await.unwrap(); + assert!( + result.status.success(), + "legacy HTTPS client failed: {}", + String::from_utf8_lossy(&result.stderr) + ); +} diff --git a/inference-router/src/sre_proxy/tests/request_boundary.rs b/inference-router/src/sre_proxy/tests/request_boundary.rs new file mode 100644 index 000000000..831c89d7a --- /dev/null +++ b/inference-router/src/sre_proxy/tests/request_boundary.rs @@ -0,0 +1,377 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::super::*; +use chrono::{Duration, Utc}; +use serde_json::{Value, json}; +use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +const TOKEN: &str = "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token"; +const REGISTRATION: &str = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical"; +const NAMESPACE: &str = "/api/v1/namespaces/kars-sre"; +const SOURCE: &str = "/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/sre"; +const ACCOUNT: &str = "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router"; +const REVIEWS: &str = "/apis/authorization.k8s.io/v1/subjectaccessreviews"; +const INVENTORY: &str = "/api/v1/namespaces/kars-sre/secrets"; +const PROJECTED_TOKEN: &str = "selected-projected-token"; +const RENEWED_TOKEN: &str = "selected-renewed-token"; + +#[derive(Clone)] +struct RecordedRequest { + method: Method, + uri: Uri, + headers: HeaderMap, + body: Value, +} + +type Requests = Arc>>; + +async fn kubernetes( + State(requests): State, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let body: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + requests.lock().unwrap().push(RecordedRequest { + method: method.clone(), + uri: uri.clone(), + headers: headers.clone(), + body: body.clone(), + }); + let expected = if uri.path() == TOKEN { + PROJECTED_TOKEN + } else { + RENEWED_TOKEN + }; + if headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + != Some(format!("Bearer {expected}").as_str()) + { + return error(StatusCode::UNAUTHORIZED, "Unexpected test credential"); + } + let value = match (method, uri.path()) { + (Method::POST, TOKEN) => json!({"status":{ + "token":RENEWED_TOKEN, + "expirationTimestamp":(Utc::now()+Duration::hours(1)).to_rfc3339(), + }}), + (Method::GET, REGISTRATION) => json!({ + "metadata":{"uid":"registration","generation":1}, + "spec":{"enabled":true,"sandbox":{"namespace":"kars-system","uid":"source"}, + "runtimeNamespace":{"uid":"namespace"}}, + "status":{"phase":"Ready","observedGeneration":1,"privacyEpoch":"epoch", + "legacySecretAccessDenied":true,"privacyRevision":crate::sre_privacy::REVISION}, + }), + (Method::GET, NAMESPACE) => json!({"metadata":{"uid":"namespace","annotations":{ + "kars.azure.com/namespace-claim-version":"v1", + "kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre", + "kars.azure.com/sandbox-uid":"source", + }}}), + (Method::GET, SOURCE) => json!({"metadata":{"uid":"source", + "annotations":{"kars.azure.com/namespace-uid":"namespace"}}}), + (Method::GET, ACCOUNT) => json!({"metadata":{"uid":"router-sa"}}), + (Method::POST, REVIEWS) => json!({"status":{"allowed":false}}), + (Method::GET, INVENTORY) => json!({ + "apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList", + "metadata":{},"items":[], + }), + _ => return error(StatusCode::NOT_FOUND, "Unexpected test Kubernetes path"), + }; + axum::Json(value).into_response() +} + +async fn attacker(State(hits): State>) -> &'static str { + hits.fetch_add(1, Ordering::SeqCst); + "attacker server reached" +} + +struct Server { + origin: String, + authority: String, + task: tokio::task::JoinHandle<()>, +} + +impl Server { + async fn start(router: Router, identity: Option<&Path>) -> Self { + let tcp = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let address = tcp.local_addr().unwrap(); + let task = if let Some(directory) = identity { + let listener = Listener { + tcp, + tls: tls(directory).unwrap(), + }; + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }) + } else { + tokio::spawn(async move { axum::serve(tcp, router).await.unwrap() }) + }; + Self { + origin: format!( + "{}://{address}", + if identity.is_some() { "https" } else { "http" } + ), + authority: address.to_string(), + task, + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +fn write_identity(directory: &Path) -> String { + let key = rcgen::KeyPair::generate().unwrap(); + let certificate = rcgen::CertificateParams::new(vec!["127.0.0.1".into(), "localhost".into()]) + .unwrap() + .self_signed(&key) + .unwrap(); + let pem = certificate.pem(); + std::fs::write(directory.join("server-cert.pem"), &pem).unwrap(); + std::fs::write(directory.join("server-key.pem"), key.serialize_pem()).unwrap(); + std::fs::write(directory.join("kube-ca.crt"), &pem).unwrap(); + pem +} + +async fn exercise_readiness_inputs(channel: &str, use_https_attacker: bool) { + let directory = tempfile::tempdir_in(".").unwrap(); + let selected = directory.path().join("selected"); + let alternate = directory.path().join("alternate"); + std::fs::create_dir(&selected).unwrap(); + std::fs::create_dir(&alternate).unwrap(); + let ca = write_identity(&selected); + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(reqwest::Certificate::from_pem(ca.as_bytes()).unwrap()) + .timeout(std::time::Duration::from_secs(10)) + .build() + .unwrap(); + let hits = Arc::new(AtomicUsize::new(0)); + let attacker_app = Router::new().fallback(attacker).with_state(hits.clone()); + let http_attacker = Server::start(attacker_app.clone(), None).await; + let https_attacker = Server::start(attacker_app, Some(&selected)).await; + + // Both attacker origins are reachable; the HTTPS attacker is trusted by + // this test CA too, so a bad destination cannot be masked by TLS rejection. + for origin in [&http_attacker.origin, &https_attacker.origin] { + assert_eq!( + client + .get(origin) + .send() + .await + .unwrap() + .text() + .await + .unwrap(), + "attacker server reached" + ); + } + assert_eq!(hits.swap(0, Ordering::SeqCst), 2); + let foreign = if use_https_attacker { + &https_attacker + } else { + &http_attacker + }; + let requests: Requests = Arc::new(Mutex::new(Vec::new())); + let kube = Server::start( + Router::new() + .fallback(kubernetes) + .with_state(requests.clone()), + Some(&selected), + ) + .await; + let configuration = json!({ + "schema":"kars.azure.com/sre-api/v1","kubeUrl":kube.origin, + "registrationUid":"registration","privacyEpoch":"epoch", + "source":{"namespace":"kars-system","name":"sre","uid":"source"}, + "runtimeNamespace":"kars-sre","namespaceUid":"namespace", + "serviceAccountUid":"router-sa","secretUid":"selected-secret", + }); + std::fs::write( + selected.join("config.json"), + serde_json::to_vec(&configuration).unwrap(), + ) + .unwrap(); + std::fs::write(selected.join("kube-token"), "initial-selected-token").unwrap(); + std::fs::write( + selected.join("kube-expires-at"), + (Utc::now() + Duration::minutes(2)).to_rfc3339(), + ) + .unwrap(); + + // Use the production loader and its HTTPS/CA/redirect configuration, not + // Backend::for_test. Only fixture setup selects this project-local volume. + let backend = Backend::load(&selected).unwrap(); + assert!(requests.lock().unwrap().is_empty()); + std::fs::write(selected.join("kube-token"), PROJECTED_TOKEN).unwrap(); + std::fs::write( + selected.join("kube-expires-at"), + (Utc::now() + Duration::minutes(4)).to_rfc3339(), + ) + .unwrap(); + std::fs::write(alternate.join("kube-token"), "attacker-alternate-token").unwrap(); + std::fs::write(alternate.join("kube-expires-at"), "invalid-attacker-expiry").unwrap(); + let mut hostile_configuration = configuration.clone(); + hostile_configuration["kubeUrl"] = foreign.origin.clone().into(); + hostile_configuration["runtimeNamespace"] = "kars-attacker".into(); + std::fs::write( + alternate.join("config.json"), + serde_json::to_vec(&hostile_configuration).unwrap(), + ) + .unwrap(); + + let proxy = Proxy { + backend, + token: Arc::from("a".repeat(64)), + capacity: Arc::new(Semaphore::new(16)), + }; + let server = Server::start(app(proxy), Some(&selected)).await; + let mut request = client.get(format!("{}/readyz", server.origin)); + let alternate_name = alternate.canonicalize().unwrap().display().to_string(); + if matches!(channel, "query" | "all") { + request = request.query(&[ + ("directory", alternate_name.as_str()), + ("credentialDirectory", alternate_name.as_str()), + ("kubeUrl", foreign.origin.as_str()), + ("runtimeNamespace", "kars-attacker"), + ]); + } + if matches!(channel, "headers" | "all") { + request = request + .header("host", &foreign.authority) + .header("x-forwarded-host", &foreign.authority) + .header( + "forwarded", + format!("host={};proto=https", foreign.authority), + ) + .header("origin", &foreign.origin) + .header("x-credential-directory", &alternate_name) + .header("x-kube-url", &foreign.origin) + .header("x-runtime-namespace", "kars-attacker"); + } + if matches!(channel, "body" | "all") { + request = request.json(&json!({ + "directory":alternate_name,"kubeUrl":foreign.origin,"runtimeNamespace":"kars-attacker", + "proxy":{"backend":{"directory":alternate_name,"config":hostile_configuration}}, + })); + } + let response = request.send().await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{channel}"); + assert_eq!( + response.json::().await.unwrap(), + json!({"ready":true}) + ); + assert_eq!(hits.load(Ordering::SeqCst), 0, "{channel}"); + + let calls = requests.lock().unwrap().clone(); + assert_eq!( + calls.len(), + 24, + "{channel}: renewal + four live identities + 18 reviews + inventory" + ); + assert_eq!(calls[0].method, Method::POST); + assert_eq!(calls[0].uri.path(), TOKEN); + assert_eq!( + calls[0].headers["authorization"], + format!("Bearer {PROJECTED_TOKEN}") + ); + assert_eq!( + calls[0].body["spec"]["boundObjectRef"], + json!({"apiVersion":"v1","kind":"Secret","name":"sre-api-router-identity","uid":"selected-secret"}) + ); + for call in &calls { + assert_eq!(call.headers["host"], kube.authority); + assert!(call.uri.query().is_none()); + assert!( + [ + TOKEN, + REGISTRATION, + NAMESPACE, + SOURCE, + ACCOUNT, + REVIEWS, + INVENTORY + ] + .contains(&call.uri.path()) + ); + assert!(!call.headers.contains_key("x-credential-directory")); + assert!(!call.headers.contains_key("x-kube-url")); + assert!(!call.headers.contains_key("x-forwarded-host")); + assert!( + !serde_json::to_string(&call.body) + .unwrap() + .contains("kars-attacker") + ); + } + for call in &calls[1..] { + assert_eq!( + call.headers["authorization"], + format!("Bearer {RENEWED_TOKEN}") + ); + } + assert_eq!( + calls.iter().filter(|call| call.uri.path() == TOKEN).count(), + 1 + ); + assert_eq!( + calls + .iter() + .filter(|call| call.uri.path() == REVIEWS) + .count(), + 18 + ); + let inventory = calls + .iter() + .find(|call| call.uri.path() == INVENTORY) + .unwrap(); + assert_eq!(inventory.method, Method::GET); + assert!( + inventory.headers["accept"] + .to_str() + .unwrap() + .contains("PartialObjectMetadataList") + ); + assert_eq!( + std::fs::read_to_string(alternate.join("kube-token")).unwrap(), + "attacker-alternate-token" + ); + assert_eq!( + std::fs::read_to_string(alternate.join("kube-expires-at")).unwrap(), + "invalid-attacker-expiry" + ); +} + +#[tokio::test] +async fn ready_hostile_inputs_cannot_select_files_origin_or_namespace() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + for channel in ["query", "headers", "body", "all"] { + for https in [false, true] { + exercise_readiness_inputs(channel, https).await; + } + } +} + +#[test] +fn production_factory_remains_startup_only_with_a_fixed_credential_directory() { + assert_eq!(DIRECTORY, "/etc/kars/sre-api"); + let production = include_str!("../mod.rs"); + assert_eq!(production.matches("Backend::load(").count(), 1); + let start = production.find("pub async fn start()").unwrap(); + let probe = production.find("pub async fn readiness_probe()").unwrap(); + let startup = &production[start..probe]; + assert!(startup.contains("let directory = PathBuf::from(DIRECTORY);")); + assert!(startup.contains("Backend::load(&directory)")); + let ready_start = production.find("async fn ready(").unwrap(); + let forward_start = production.find("async fn forward(").unwrap(); + let ready = &production[ready_start..forward_start]; + assert!(ready.contains("async fn ready(State(proxy): State)")); + assert!(!ready.contains("Backend::load(")); + assert!(include_str!("../../main.rs").contains("kars_inference_router::sre_proxy::start()")); +} diff --git a/mesh-plugin/package-lock.json b/mesh-plugin/package-lock.json index fc0356adc..3aa6dd8c5 100644 --- a/mesh-plugin/package-lock.json +++ b/mesh-plugin/package-lock.json @@ -80,28 +80,6 @@ "node": ">=18.0.0" } }, - "node_modules/@microsoft/agent-governance-sdk/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -834,6 +812,28 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", diff --git a/mesh-plugin/package.json b/mesh-plugin/package.json index 75b5adb32..08bac40c3 100644 --- a/mesh-plugin/package.json +++ b/mesh-plugin/package.json @@ -48,7 +48,7 @@ "ws": "^8.21.0" }, "overrides": { - "js-yaml": "4.3.1" + "js-yaml": "4.3.2" }, "devDependencies": { "@types/node": "^25.6.0", diff --git a/runtimes/openclaw/package-lock.json b/runtimes/openclaw/package-lock.json index fe71a1a42..831f42762 100644 --- a/runtimes/openclaw/package-lock.json +++ b/runtimes/openclaw/package-lock.json @@ -38,6 +38,7 @@ "devDependencies": { "@types/node": "^25.6.0", "@types/ws": "^8", + "oxlint": "^0.16.0", "typescript": "^5.7", "vitest": "^4.1.8" }, @@ -549,29 +550,6 @@ "node": ">=18.0.0" } }, - "node_modules/@microsoft/agent-governance-sdk/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", @@ -1350,6 +1328,29 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", diff --git a/runtimes/openclaw/package.json b/runtimes/openclaw/package.json index 7096a8d8d..5f7654afc 100644 --- a/runtimes/openclaw/package.json +++ b/runtimes/openclaw/package.json @@ -51,6 +51,6 @@ }, "overrides": { "esbuild": "^0.28.1", - "js-yaml": "4.3.1" + "js-yaml": "4.3.2" } } \ No newline at end of file diff --git a/shared/sre_privacy.rs b/shared/sre_privacy.rs new file mode 100644 index 000000000..dec473751 --- /dev/null +++ b/shared/sre_privacy.rs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Shared wire contract for controller issuance and private-router authorization. +//! Included by both binaries without another dependency or feature variant. + +use serde_json::{Value, json}; + +pub const REVISION: &str = "kars.azure.com/sre-privacy/v2"; + +pub fn secret_access_reviews(namespace: &str) -> Vec { + let mut reviews = Vec::new(); + for scope in [Some(namespace), None] { + for verb in ["get", "list", "watch"] { + for name in [ + None, + Some("router-services-admin"), + Some("sre-api-router-identity"), + ] { + let mut attributes = json!({"group":"","resource":"secrets","verb":verb}); + if let Some(scope) = scope { + attributes["namespace"] = scope.into(); + } + if let Some(name) = name { + attributes["name"] = name.into(); + } + reviews.push(json!({ + "apiVersion":"authorization.k8s.io/v1","kind":"SubjectAccessReview", + "spec":{"user":"system:serviceaccount:kars-sre:sandbox", + "groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:kars-sre"], + "resourceAttributes":attributes}, + })); + } + } + } + reviews +} + +pub fn require_denial(response: &Value) -> Result<(), &'static str> { + if response["status"]["allowed"].as_bool() != Some(false) + || response["status"] + .get("evaluationError") + .is_some_and(|error| !error.is_null() && error.as_str() != Some("")) + { + return Err("Legacy SRE Secret get/list/watch authorization is allowed or indeterminate"); + } + Ok(()) +} + +/// Only metadata is requested. Even an Opaque Secret carrying this reserved +/// annotation is quarantined: no type mutation or stale SA-UID alias is adopted. +pub fn reject_legacy_aliases(list: &Value, account_uids: &[&str]) -> Result<(), &'static str> { + let items = list["items"] + .as_array() + .ok_or("SRE credential metadata inventory is invalid")?; + if list["metadata"] + .get("continue") + .is_some_and(|token| !token.is_null() && token.as_str() != Some("")) + { + return Err("SRE credential metadata inventory is incomplete"); + } + for item in items { + let metadata = &item["metadata"]; + if ["name", "uid", "resourceVersion"] + .iter() + .any(|key| metadata[*key].as_str().is_none_or(str::is_empty)) + { + return Err("SRE credential metadata omitted its exact identity"); + } + let annotations = &metadata["annotations"]; + if annotations["kubernetes.io/service-account.name"] == "sre-api-router" + || annotations["kubernetes.io/service-account.uid"] + .as_str() + .is_some_and(|uid| !uid.is_empty() && account_uids.contains(&uid)) + { + return Err( + "Unsafe legacy SRE token Secret alias exists; operator review required, no Secret adopted or deleted", + ); + } + } + Ok(()) +} diff --git a/tests/e2e/namespace-ownership.sh b/tests/e2e/namespace-ownership.sh index 3972742d4..31f475c2f 100644 --- a/tests/e2e/namespace-ownership.sh +++ b/tests/e2e/namespace-ownership.sh @@ -2,60 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# Sourced only by the existing disposable Kind harness. +# Retains the original SRE source/namespace lifecycle gate, now using the +# required safe CLI stage/review/enroll/migrate/install/retire/uninstall flow. +# The fresh source is created after legacy retirement and receives new UIDs; +# retained registration history is updated with exact reviewed UID/RV CAS. test_sre_namespace_ownership() { - local context="kind-kars-e2e" system_uid sandbox_uid namespace_uid claimed_uid backlink source_ns - local k=(kubectl --context "$context") - system_uid=$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}') || { - fail "Cannot read the disposable Kind namespace"; return 1; - } - if ! helm upgrade kars "$ROOT_DIR/deploy/helm/kars" \ - --kube-context "$context" --namespace kars-system --reuse-values \ - --set sre.enabled=true --set-string runtimes.hermes.image=kars-sandbox-e2e:dev \ - --wait --timeout 3m; then - fail "Fresh SRE enable failed"; return 1; - fi - local deadline=$(($(date +%s) + 120)) - local materialized=0 - while [ "$(date +%s)" -lt "$deadline" ]; do - if "${k[@]}" get deployment sre -n kars-sre >/dev/null 2>&1 \ - && "${k[@]}" get serviceaccount sre-writer -n kars-sre >/dev/null 2>&1; then - materialized=1 - break - fi - sleep 2 - done - if [ "$materialized" -ne 1 ]; then - "${k[@]}" get karssandbox sre -n kars-system -o yaml || true - fail "SRE did not materialize its deployment and writer account after namespace claiming" - return 1 - fi - sandbox_uid=$("${k[@]}" get karssandbox sre -n kars-system -o jsonpath='{.metadata.uid}') || return 1 - namespace_uid=$("${k[@]}" get namespace kars-sre -o jsonpath='{.metadata.uid}') || return 1 - claimed_uid=$("${k[@]}" get namespace kars-sre -o go-template='{{index .metadata.annotations "kars.azure.com/sandbox-uid"}}') || return 1 - source_ns=$("${k[@]}" get namespace kars-sre -o go-template='{{index .metadata.annotations "kars.azure.com/sandbox-namespace"}}') || return 1 - backlink=$("${k[@]}" get karssandbox sre -n kars-system -o go-template='{{index .metadata.annotations "kars.azure.com/namespace-uid"}}') || return 1 - if [ -z "$sandbox_uid" ] || [ -z "$namespace_uid" ] \ - || [ "$claimed_uid" != "$sandbox_uid" ] || [ "$backlink" != "$namespace_uid" ] \ - || [ "$source_ns" != "kars-system" ]; then - fail "Fresh SRE runtime lacks exact two-way namespace ownership"; return 1; - fi - if [ "$("${k[@]}" get serviceaccount sre-writer -n kars-sre -o jsonpath='{.automountServiceAccountToken}')" != "false" ]; then - fail "SRE writer account does not disable token automount"; return 1; - fi - pass "Fresh SRE install materializes a UID-bound namespace, deployment and non-automounting writer" - - if ! helm upgrade kars "$ROOT_DIR/deploy/helm/kars" \ - --kube-context "$context" --namespace kars-system --reuse-values \ - --set sre.enabled=false --wait --timeout 3m; then - fail "SRE disable failed"; return 1; - fi - if ! "${k[@]}" wait --for=delete karssandbox/sre -n kars-system --timeout=120s \ - || ! "${k[@]}" wait --for=delete namespace/kars-sre --timeout=120s; then - fail "SRE namespace cleanup did not complete"; return 1; - fi - if [ "$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}')" != "$system_uid" ]; then - fail "SRE removal changed the core namespace identity"; return 1; - fi - pass "SRE removal completes guarded cleanup and preserves the core namespace" + sre_authority_phase fresh } diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 9e9c58cce..96fe89da6 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -28,6 +28,8 @@ CLUSTER_NAME="kars-e2e" RUNTIME="${KARS_E2E_RUNTIME:-openclaw}" PASS=0 FAIL=0 +SRE_LEGACY_PREPARED=0 +E2E_KUBECONFIG="$ROOT_DIR/.e2e-kind-kubeconfig" # ─── Colors ─────────────────────────────────────────────────────────────────── RED='\033[0;31m' @@ -44,12 +46,18 @@ warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } setup_cluster() { info "Creating Kind cluster: $CLUSTER_NAME" - if kind get clusters 2>/dev/null | grep -q "$CLUSTER_NAME"; then + if kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then info "Cluster already exists, reusing" + kind get kubeconfig --name "$CLUSTER_NAME" > "$E2E_KUBECONFIG" + chmod 600 "$E2E_KUBECONFIG" + export KUBECONFIG="$E2E_KUBECONFIG" return fi kind create cluster --name "$CLUSTER_NAME" --config "$SCRIPT_DIR/kind-config.yaml" + kind get kubeconfig --name "$CLUSTER_NAME" > "$E2E_KUBECONFIG" + chmod 600 "$E2E_KUBECONFIG" + export KUBECONFIG="$E2E_KUBECONFIG" info "Cluster created" } @@ -151,6 +159,7 @@ install_crds() { # KARS_E2E_* env vars; the defaults here cover local runs. local replicas="${KARS_E2E_CONTROLLER_REPLICAS:-1}" local disable_le="${KARS_E2E_DISABLE_LEADER_ELECTION:-1}" + local helm_wait_arg=--wait local extra_set_args=( --set "controller.replicas=${replicas}" --set "inferenceRouter.replicas=${replicas}" @@ -171,6 +180,18 @@ install_crds() { --set-string "controller.extraEnv[0].value=false" ) fi + if [ "$SRE_LEGACY_PREPARED" = "1" ]; then + # Legacy fixtures predate the new admission policies. Explicit CLI + # authority staging already installed those APIs with controller=0; + # start the qualified controller while retaining the reviewed shapes. + extra_set_args+=(--set sre.enabled=true --set sre.authorityStage=true + --set-string runtimes.hermes.image=kars-sandbox-e2e:dev) + # Consumer policy acknowledgements cannot become Ready before explicit + # enrollment below. Wait for built-ins here; migration remains fatal. + local helm_version + helm_version=$(helm version --template '{{.Version}}') || return 1 + helm_wait_arg=$(sre_migration_helm_wait_arg "$helm_version") || return 1 + fi if ! helm upgrade --install kars "$ROOT_DIR/deploy/helm/kars" \ --namespace kars-system \ --create-namespace \ @@ -183,18 +204,20 @@ install_crds() { --set sandbox.image.repository=kars-sandbox-e2e \ --set sandbox.image.tag=dev \ "${extra_set_args[@]}" \ - --wait --timeout 5m; then + "$helm_wait_arg" --timeout 5m; then warn "Helm install did not converge within 5m — dumping diagnostics" + PYTHONPATH="$ROOT_DIR/tests/e2e" python3 -m sre_authority.bootstrap_probe --diagnostics-only \ + || warn "Bounded controller admission diagnostics were incomplete" kubectl get all -n kars-system || true - kubectl describe pod -n kars-system -l app.kubernetes.io/component=controller || true - kubectl logs -n kars-system -l app.kubernetes.io/component=controller --tail=200 || true return 1 fi } teardown() { + sre_authority_cleanup || true info "Tearing down Kind cluster" kind delete cluster --name "$CLUSTER_NAME" 2>/dev/null || true + rm -f "$E2E_KUBECONFIG" } # ─── Tests ──────────────────────────────────────────────────────────────────── @@ -291,7 +314,7 @@ EOF } test_networkpolicy_created() { - if kubectl get networkpolicy -n kars-e2e-test sandbox-policy --no-headers 2>/dev/null | grep -q sandbox-policy; then + if wait_for_resource networkpolicy sandbox-policy kars-e2e-test 30; then pass "NetworkPolicy created in sandbox namespace" else fail "NetworkPolicy not found" @@ -299,7 +322,7 @@ test_networkpolicy_created() { } test_serviceaccount_created() { - if kubectl get serviceaccount -n kars-e2e-test sandbox --no-headers 2>/dev/null | grep -q sandbox; then + if wait_for_resource serviceaccount sandbox kars-e2e-test 30; then pass "ServiceAccount created in sandbox namespace" else fail "ServiceAccount not found" @@ -1318,8 +1341,8 @@ EOF test_runtime_hermes() { # KarsSandbox of kind Hermes should be processed by the controller: # plan_hermes dispatches, namespace is created, and the agent - # container's image carries the hermes runtime tag (kars-runtime-hermes) - # rather than the OpenClaw default. Mirrors test_runtime_anthropic and + # container uses the configured Hermes image (including the SRE fixture's + # explicit stand-in pin), with actual Hermes runtime dispatch. Mirrors test_runtime_anthropic and # follows the same tolerance pattern: Deployment may not materialize # if there's no real InferencePolicy provider in this lane, so the # image-tag assertion is diag-only when no Deployment is present. @@ -1349,14 +1372,24 @@ EOF echo "" fail "Hermes runtime: no namespace" fi - local image + local image configured runtime_kind image=$(kubectl get deploy -n kars-e2e-hermes e2e-hermes -o jsonpath='{.spec.template.spec.containers[?(@.name=="agent")].image}' 2>/dev/null || true) if [ -n "$image" ]; then - if echo "$image" | grep -qE "hermes|kars-runtime-hermes"; then - pass "Hermes Deployment uses hermes runtime image ($image)" + if ! configured=$(kubectl get deployment kars-controller -n kars-system \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="controller")].env[?(@.name=="HERMES_RUNTIME_IMAGE")].value}'); then + fail "Hermes runtime: could not inspect the configured image" + return + fi + if ! runtime_kind=$(kubectl get deployment e2e-hermes -n kars-e2e-hermes \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="agent")].env[?(@.name=="KARS_RUNTIME_KIND")].value}'); then + fail "Hermes runtime: could not inspect actual runtime dispatch" + return + fi + if sre_hermes_image_matches "$image" "$configured" "$runtime_kind"; then + pass "Hermes Deployment preserves the configured image and Hermes runtime dispatch ($image)" else echo " [diag] container image: $image" - fail "Hermes Deployment image does not reference hermes runtime" + fail "Hermes Deployment does not match its configured image or runtime dispatch" fi else echo " [diag] no Deployment yet (likely no InferencePolicy provider in this lane)" @@ -3015,21 +3048,31 @@ EOF # ─── Main ───────────────────────────────────────────────────────────────────── +source "$SCRIPT_DIR/sre-authority.sh" source "$SCRIPT_DIR/namespace-ownership.sh" source "$SCRIPT_DIR/credential-sources.sh" main() { + umask 077 echo "" echo "═══════════════════════════════════════════════════════" echo " kars E2E Test Suite (runtime: $RUNTIME)" echo "═══════════════════════════════════════════════════════" echo "" + PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$SCRIPT_DIR" \ + python3 -m unittest discover -s "$SCRIPT_DIR/sre_authority" -p '*_test.py' trap teardown EXIT setup_cluster + # Validate the public cluster API before any Rust images or private fixtures. + PYTHONDONTWRITEBYTECODE=1 python3 "$SCRIPT_DIR/sre_authority/registration_schema.py" build_images + prepare_sre_authority_legacy install_crds + # Finish real legacy retirement before unrelated tests can create private + # namespaces/credentials. Failure is fatal, not a skipped/false-positive gate. + test_sre_authority_migration echo "" info "Running tests..." diff --git a/tests/e2e/sre-authority.py b/tests/e2e/sre-authority.py new file mode 100644 index 000000000..8a68639d1 --- /dev/null +++ b/tests/e2e/sre-authority.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import argparse +import json +from pathlib import Path + + +def cleanup(): + work = Path(__file__).resolve().parents[2] / ".e2e-sre-authority" + # Remove only explicitly owned credential files, never broad namespaces, + # arbitrary directories, or another process's kubeconfig. + for name in ("admin.json", "registrar.json", "tenant.json", "normal.json", + "old-agent.json", "watch-only.json", "unrelated.json", "opaque.json", "admin-key.pem", + "admin-cert.pem", "api-ca.pem", "agent/token", "agent/ca.crt", "agent/namespace"): + (work / name).unlink(missing_ok=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("phase", choices=("prepare", "legacy", "fresh", "cleanup")) + options = parser.parse_args() + if options.phase == "cleanup": + cleanup() + return 0 + from sre_authority.common import Harness, SYSTEM + from sre_authority.fixtures import CONTROL, CONTROL_NS, prepare_legacy + from sre_authority.migration import fresh_reenrollment, legacy_migration, retire_and_uninstall + from sre_authority.proxy import proxy_acceptance + harness = None + try: + harness = Harness(options.phase) + if options.phase == "prepare": + prepare_legacy(harness) + elif options.phase == "legacy": + legacy_migration(harness) + proxy_acceptance(harness) + retire_and_uninstall(harness) + source = harness.get("karssandbox", CONTROL, SYSTEM) + if source: + harness.api("DELETE", f"/apis/kars.azure.com/v1alpha1/namespaces/{SYSTEM}/karssandboxes/{CONTROL}", + body={"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": { + "uid": harness.state["control_source_uid"], + "resourceVersion": source["metadata"]["resourceVersion"]}}, status=(200, 202)) + harness.poll("owned control fixture cleanup", lambda: harness.get("namespace", CONTROL_NS) is None, seconds=120) + else: + fresh_reenrollment(harness) + proxy_acceptance(harness) + retire_and_uninstall(harness) + harness.save() + return 0 + except Exception as error: + # Report only the exception class and controlled assertions. API bodies, + # command output, JWTs and TLS private keys never become failure logs. + print(f"SRE-FAIL {options.phase}: {str(error) if isinstance(error, AssertionError) else type(error).__name__}", flush=True) + if harness: + from sre_authority.bootstrap_diagnostics import exception_summary + try: + print("SRE-ERROR", json.dumps(exception_summary(error, harness.root)), flush=True) + except Exception: + print("SRE-ERROR sanitized exception summary unavailable", flush=True) + harness.diagnostics() + return 1 + finally: + if harness: + harness.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/sre-authority.sh b/tests/e2e/sre-authority.sh new file mode 100644 index 000000000..db4e005c9 --- /dev/null +++ b/tests/e2e/sre-authority.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Called only by the explicit disposable Kind harness. No live/default context. +sre_migration_helm_wait_arg() { + case "$1" in + v3.*) printf '%s\n' --wait ;; + v4.*) printf '%s\n' --wait=legacy ;; + *) printf '%s\n' "Unsupported Helm version for staged SRE acceptance" >&2; return 1 ;; + esac +} + +sre_hermes_image_matches() { + local image="$1" configured="$2" runtime="$3" + [ "$runtime" = "Hermes" ] && [ -n "$image" ] || return 1 + if [ -n "$configured" ]; then + [ "$image" = "$configured" ] + else + [[ "$image" == *hermes* ]] + fi +} + +sre_authority_phase() { + local phase="$1" output result=0 line + info "SRE authority acceptance: ${phase}" + output=$(python3 "$SCRIPT_DIR/sre-authority.py" "$phase") || result=$? + while IFS= read -r line; do + case "$line" in + "SRE-PASS "*) pass "${line#SRE-PASS }" ;; + "SRE-FAIL "*) fail "${line#SRE-FAIL }" ;; + *) [ -z "$line" ] || printf '%s\n' "$line" ;; + esac + done <<< "$output" + return "$result" +} + +prepare_sre_authority_legacy() { + sre_authority_phase prepare || return + SRE_LEGACY_PREPARED=1 +} + +test_sre_authority_migration() { + sre_authority_phase legacy +} + +sre_authority_cleanup() { + python3 "$SCRIPT_DIR/sre-authority.py" cleanup +} diff --git a/tests/e2e/sre_authority/__init__.py b/tests/e2e/sre_authority/__init__.py new file mode 100644 index 000000000..732a1699c --- /dev/null +++ b/tests/e2e/sre_authority/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Hosted disposable-Kind acceptance, not an agent/model execution harness.""" diff --git a/tests/e2e/sre_authority/admission.py b/tests/e2e/sre_authority/admission.py new file mode 100644 index 000000000..c503e0bb0 --- /dev/null +++ b/tests/e2e/sre_authority/admission.py @@ -0,0 +1,272 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json + +from .common import POLICIES, PRIVATE, REGISTRATION, RUNTIME, STANDIN, TENANT, assert_denial, require +from .collection_delete_probe import stable_object + + +def policies_ready(h): + def checked(): + for suffix in POLICIES: + name = f"kars-sre-{suffix}" + policy = h.get("validatingadmissionpolicy", name) + binding = h.get("validatingadmissionpolicybinding", name) + if not policy or not binding: + return False + status = policy.get("status", {}) + if status.get("observedGeneration") != policy["metadata"]["generation"] or "typeChecking" not in status: + return False + require(not status["typeChecking"].get("expressionWarnings"), f"CEL type warnings in {name}") + require(policy["spec"].get("failurePolicy") == "Fail", f"{name} does not fail closed") + require(binding["spec"]["policyName"] == name and "Deny" in binding["spec"]["validationActions"], + f"{name} lacks a matching enforcing binding") + return True + h.poll("all observed/enforcing SRE CEL policies", checked, seconds=90) + h.passed(f"All {len(POLICIES)} real API admission policies are observed, CEL-type-checked without warnings, and Deny-bound") + + +def pod_spec(private=True): + spec = {"serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "securityContext": {"runAsNonRoot": True, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "probe", "image": STANDIN, "command": ["/bin/sh", "-c", "sleep infinity"], + "securityContext": {"allowPrivilegeEscalation": False, "capabilities": {"drop": ["ALL"]}}}]} + if private: + spec["volumes"] = [{"name": "private", "secret": {"secretName": PRIVATE}}] + spec["containers"][0]["volumeMounts"] = [{"name": "private", "mountPath": "/private", "readOnly": True}] + return spec + + +def reserved_source_probe(): + return {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": "sre", "namespace": TENANT}, + "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1"}}, + "inferenceRef": {"name": "sre-inference"}, + "sandbox": {"isolation": "standard"}}} + + +def rc_update_preview(h, path, original, candidate): + current = h.api("GET", path, status=200).json() + require(stable_object(current) == stable_object(original) + and not current["metadata"].get("deletionTimestamp"), + "ReplicationController update target changed beyond status") + for attempt in range(3): + body = copy.deepcopy(candidate) + body["metadata"] = copy.deepcopy(current["metadata"]) + response = h.api("PUT", path + "?dryRun=All", body=body, user="tenant") + if response.status_code != 409 or response.json().get("reason") != "Conflict" or attempt == 2: + return response + refreshed = h.api("GET", path, status=200).json() + require(stable_object(refreshed) == stable_object(current) + and not refreshed["metadata"].get("deletionTimestamp"), + "ReplicationController update conflict changed more than status; no retry") + if refreshed["metadata"]["resourceVersion"] == current["metadata"]["resourceVersion"]: + return response + current = refreshed + raise AssertionError("ReplicationController preview exceeded its bounded attempts") + + +def replication_controller_cases(h): + collection = f"/api/v1/namespaces/{RUNTIME}/replicationcontrollers" + for resource, verb, namespace, subresource, expected in [ + ("replicationcontrollers", "create", RUNTIME, "", True), + ("replicationcontrollers", "update", RUNTIME, "", True), + ("pods", "get", RUNTIME, "log", True), + ("pods", "create", "", "", False), + ("secrets", "get", RUNTIME, "", False), + ]: + response = h.api("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", + user="tenant", status=201, body={ + "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", + "spec": {"resourceAttributes": { + "group": "", "resource": resource, "verb": verb, "namespace": namespace, + **({"subresource": subresource} if subresource else {})}}, + }).json()["status"] + require(response.get("allowed") is expected and not response.get("evaluationError"), + "ReplicationController probe does not have the intended limited authority") + + pod = pod_spec(False) + pod["schedulerName"] = "kars-e2e-admission-never-schedule" + pod["containers"][0]["command"] = ["/bin/true"] + pod["containers"][0]["imagePullPolicy"] = "Never" + ordinary = { + "apiVersion": "v1", "kind": "ReplicationController", + "metadata": {"name": "e2e-rc-boundary", "namespace": RUNTIME}, + "spec": {"replicas": 0, "selector": {"app": "e2e-rc-boundary"}, + "template": {"metadata": {"labels": {"app": "e2e-rc-boundary"}}, "spec": pod}}, + } + without_template = copy.deepcopy(ordinary) + without_template["spec"].pop("template") + invalid = h.api("POST", collection + "?dryRun=All", body=without_template, + user="tenant", status=422).json() + require(invalid.get("reason") == "Invalid" and any( + cause.get("field") == "spec.template" and cause.get("reason") == "FieldValueRequired" + for cause in invalid.get("details", {}).get("causes", [])), + "Template-less ReplicationController did not receive the native required-field rejection") + created = h.api("POST", collection, body=ordinary, user="tenant", status=201).json() + identity = created["metadata"]["uid"] + path = collection + "/" + created["metadata"]["name"] + primary_failure = False + try: + for variant in ["ordinary", "secret-volume", "projected-secret", "env", "env-key", + "init-env", "service-account"]: + current = h.api("GET", path, status=200).json() + require(current["metadata"]["uid"] == identity and current["spec"] == created["spec"], + "ReplicationController fixture identity or template changed") + candidate = copy.deepcopy(current) + candidate.pop("status", None) + spec = candidate["spec"]["template"]["spec"] + if variant == "secret-volume": + spec["volumes"] = [{"name": "private", "secret": {"secretName": PRIVATE}}] + elif variant == "projected-secret": + spec["volumes"] = [{"name": "private", "projected": {"sources": [{"secret": { + "name": PRIVATE}}]}}] + elif variant == "env": + spec["containers"][0]["envFrom"] = [{"secretRef": {"name": PRIVATE}}] + elif variant == "env-key": + spec["containers"][0]["env"] = [{"name": "PRIVATE_TOKEN", "valueFrom": { + "secretKeyRef": {"name": PRIVATE, "key": "kube-token"}}}] + elif variant == "init-env": + initializer = copy.deepcopy(spec["containers"][0]) + initializer["name"] = "private-init" + initializer["envFrom"] = [{"secretRef": {"name": PRIVATE}}] + spec["initContainers"] = [initializer] + elif variant == "service-account": + spec["serviceAccountName"] = "sre-api-router" + for method, target in [("POST", collection), ("PUT", path)]: + body = copy.deepcopy(candidate) + if method == "POST": + body["metadata"] = {"name": "e2e-rc-dry-run", "namespace": RUNTIME} + response = (rc_update_preview(h, target, current, body) if method == "PUT" + else h.api(method, target + "?dryRun=All", body=body, user="tenant")) + print("SRE-RC-ADMISSION " + json.dumps({ + "variant": variant, "method": method, "httpStatus": response.status_code, + }), flush=True) + if variant == "ordinary": + require(response.status_code == (201 if method == "POST" else 200), + f"Ordinary ReplicationController {method} was not admitted: HTTP {response.status_code}") + else: + assert_denial(response, f"{method} ReplicationController {variant}", + "kars-sre-private-workloads") + except BaseException: + primary_failure = True + raise + finally: + try: + removed = False + for _ in range(3): + current = h.api("GET", path, status=(200, 404)) + if current.status_code == 404: + removed = True + break + current = current.json() + require(current["metadata"]["uid"] == identity and current["spec"] == created["spec"], + "ReplicationController cleanup refuses changed fixture ownership or template") + response = h.api("DELETE", path, body={ + "apiVersion": "v1", "kind": "DeleteOptions", "preconditions": { + "uid": identity, "resourceVersion": current["metadata"]["resourceVersion"]}, + }) + if response.status_code == 409: + continue + require(response.status_code in (200, 202), "Owned ReplicationController cleanup failed") + removed = True + break + require(removed, "Owned ReplicationController cleanup contention exceeded its bound") + def gone(): + response = h.api("GET", path, status=(200, 404)) + if response.status_code == 404: + return True + require(response.json()["metadata"]["uid"] == identity, + "ReplicationController was replaced during cleanup") + return False + h.poll("owned ReplicationController fixture deletion", gone, seconds=30) + except Exception: + if not primary_failure: + raise + print("SRE-DIAG owned ReplicationController fixture cleanup failed", flush=True) + h.passed("Real ReplicationController CREATE/UPDATE deny private mounts and identities under namespaced-only authority") + + +def admission_cases(h, enrollment): + registration = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, "spec": enrollment} + # Normal SA has no cluster enrollment permission. Tenant probe has CREATE + # but deliberately lacks USE, so the second check exercises actual CEL. + assert_denial(h.api("POST", REGISTRATION.rsplit("/", 1)[0] + "?dryRun=All", + body=registration, user="normal"), "ordinary registration RBAC") + assert_denial(h.api("POST", REGISTRATION.rsplit("/", 1)[0] + "?dryRun=All", + body=registration, user="tenant"), "registration CEL", "kars-sre-registration-authority") + review = h.api("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", user="normal", body={ + "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", + "spec": {"resourceAttributes": {"group": "kars.azure.com", "resource": "karssreregistrations", + "name": "canonical", "verb": "use"}}}, status=201).json() + require(review["status"]["allowed"] is False, "Ordinary SA unexpectedly has registrar use") + h.passed("Real RBAC and CEL independently deny ordinary/tenant registration and registrar use") + + source = reserved_source_probe() + assert_denial(h.api("POST", f"/apis/kars.azure.com/v1alpha1/namespaces/{TENANT}/karssandboxes?dryRun=All", + body=source, user="tenant"), "reserved source", "kars-sre-source-authority") + source["metadata"]["name"] = "not-canonical" + source["metadata"]["labels"] = {"kars.azure.com/role": "sre"} + assert_denial(h.api("POST", f"/apis/kars.azure.com/v1alpha1/namespaces/{TENANT}/karssandboxes?dryRun=All", + body=source, user="tenant"), "reserved SRE role", "kars-sre-source-authority") + h.passed("Namespace authority cannot create a reserved-name or SRE-labelled source") + + sa = {"apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": "sre-api-router", "namespace": TENANT}} + assert_denial(h.api("POST", f"/api/v1/namespaces/{TENANT}/serviceaccounts?dryRun=All", + body=sa, user="tenant"), "reserved ServiceAccount", "kars-sre-private-identity") + secret = {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": PRIVATE, "namespace": RUNTIME}, + "stringData": {"kube-token": "not-a-credential"}} + assert_denial(h.api("POST", f"/api/v1/namespaces/{RUNTIME}/secrets?dryRun=All", + body=secret, user="tenant"), "reserved material", "kars-sre-private-material") + h.passed("Real admission denies reserved identities and private material despite namespaced create permission") + + # An otherwise valid ordinary Pod is admitted in server-side dry-run; + # private variants must be denied by the intended policy, not schema/RBAC. + ordinary = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "e2e-ordinary", "namespace": RUNTIME}, "spec": pod_spec(False)} + h.api("POST", f"/api/v1/namespaces/{RUNTIME}/pods?dryRun=All", body=ordinary, user="tenant", status=201) + private = {**ordinary, "metadata": {"name": "e2e-private", "namespace": RUNTIME}, "spec": pod_spec()} + assert_denial(h.api("POST", f"/api/v1/namespaces/{RUNTIME}/pods?dryRun=All", + body=private, user="tenant"), "private Pod mount", "kars-sre-private-mounts") + env_only = pod_spec(False) + env_only["containers"][0]["envFrom"] = [{"secretRef": {"name": PRIVATE}}] + assert_denial(h.api("POST", f"/api/v1/namespaces/{RUNTIME}/pods?dryRun=All", + body={**private, "spec": env_only}, user="tenant"), "private envFrom", "kars-sre-private-mounts") + h.passed("A valid ordinary Pod is accepted; private volume/envFrom laundering is denied") + + for kind, group, plural in [("Deployment", "apps/v1", "deployments"), ("ReplicaSet", "apps/v1", "replicasets"), + ("Job", "batch/v1", "jobs"), ("CronJob", "batch/v1", "cronjobs")]: + template = {"metadata": {"labels": {"app": "e2e-private-template"}}, "spec": pod_spec()} + if kind in ("Job", "CronJob"): + template["spec"]["restartPolicy"] = "Never" + spec = {"template": template} + if kind in ("Deployment", "ReplicaSet"): + spec.update({"replicas": 1, "selector": {"matchLabels": template["metadata"]["labels"]}}) + if kind == "CronJob": + spec = {"schedule": "0 * * * *", "jobTemplate": {"spec": spec}} + obj = {"apiVersion": group, "kind": kind, "metadata": {"name": "e2e-private-template", "namespace": RUNTIME}, "spec": spec} + policy = "kars-sre-private-cronjobs" if kind == "CronJob" else "kars-sre-private-workloads" + assert_denial(h.api("POST", f"/apis/{group}/namespaces/{RUNTIME}/{plural}?dryRun=All", + body=obj, user="tenant"), f"{kind} private template", policy) + h.passed(f"Real {kind} admission prevents private-material laundering through workload controllers") + replication_controller_cases(h) + + +def runtime_denials(h, pod): + response = h.api("POST", f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router/token", + body={"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest", + "spec": {"audiences": [], "expirationSeconds": 600}}, user="tenant") + assert_denial(response, "private TokenRequest", "kars-sre-private-identity") + for subresource, args in [ + ("exec", ["exec", "-n", RUNTIME, pod, "-c", "agent", "--", "/bin/true"]), + ("attach", ["attach", "-n", RUNTIME, pod, "-c", "agent"]), + ("portforward", ["port-forward", "-n", RUNTIME, f"pod/{pod}", "19446:9446"]), + ]: + result = h.k(*args, user="tenant", expected=None, timeout=25) + require(result.returncode != 0 and "kars-sre-private-connect" in result.stderr + and "forbidden" in result.stderr.lower(), f"{subresource} did not receive the intended connect-admission denial") + assert_denial(h.api("GET", f"/api/v1/namespaces/{RUNTIME}/pods/{pod}:9446/proxy/api/v1/namespaces", + user="tenant"), "Pod proxy connect", "kars-sre-private-connect") + h.passed("Tenant TokenRequests and actual exec/attach/port-forward/proxy attempts receive intended admission denials") diff --git a/tests/e2e/sre_authority/binding_probe.py b/tests/e2e/sre_authority/binding_probe.py new file mode 100644 index 000000000..0ece4efcc --- /dev/null +++ b/tests/e2e/sre_authority/binding_probe.py @@ -0,0 +1,342 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Image-free, real-controller-principal RBAC retirement dry-run experiment.""" + +import base64 +import json +import re +import ssl +import time +from urllib.error import HTTPError +from urllib.parse import urlsplit +from urllib.request import HTTPSHandler, ProxyHandler, Request, build_opener + +from .common import POLICIES, REGISTRATION, RUNTIME, SYSTEM, require +from .fixtures import LEGACY_COMMIT +from .registration_schema import CONTEXT, command, request + +RBAC = "/apis/rbac.authorization.k8s.io/v1" +READER = "kars-sre-reader" +CUSTOM = "e2e-sre-custom-review" +EXTRA = "e2e-sre-reader-bind-proof" +CONTROLLER = f"system:serviceaccount:{SYSTEM}:kars-controller" +RETIRED = "kars.azure.com/sre-legacy-retired" +SURVIVOR = {"kind": "ServiceAccount", "name": "e2e-retained-subject", "namespace": SYSTEM} +LEGACY = {"kind": "ServiceAccount", "name": "sandbox", "namespace": RUNTIME} + + +def create(port, path, obj): + code, result = request(port, "POST", path, obj) + require(code == 201 and result.get("metadata", {}).get("uid") + and result["metadata"].get("resourceVersion"), "RBAC proof fixture CREATE failed") + return result + + +def get(port, path): + code, obj = request(port, "GET", path) + require(code == 200 and isinstance(obj, dict), "RBAC proof fixture GET failed") + return obj + + +def historical_reader(root): + from .bootstrap_probe import converted_objects + command("reader-history", ["git", "fetch", "--no-tags", "--depth=1", + "https://github.com/Azure/kars.git", LEGACY_COMMIT], root=root) + chart = root / ".e2e-sre-reader-chart" + require(not chart.exists(), "Refusing to overwrite an existing historical reader fixture directory") + files = ("Chart.yaml", "values.yaml", "templates/sre.yaml") + try: + for name in files: + contents = command("reader-source", ["git", "show", + f"{LEGACY_COMMIT}:deploy/helm/kars/{name}"], root=root) + path = chart / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents) + rendered = command("reader-render", ["helm", "template", "kars", str(chart), + "--namespace", SYSTEM, "--set", "sre.enabled=true", "--show-only", "templates/sre.yaml"], root=root) + documents = [document for document in re.split(r"(?m)^---\s*\n", rendered) + if re.search(r"(?m)^kind: ClusterRole\s*$", document) + and re.search(r"(?m)^ name: kars-sre-reader\s*$", document)] + require(len(documents) == 1, "Pinned legacy chart did not emit exactly one reader role") + objects = converted_objects(command("reader-convert", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=documents[0])) + require(len(objects) == 1 and objects[0]["metadata"]["name"] == READER + and objects[0]["kind"] == "ClusterRole", "Unexpected historical RBAC fixture") + return objects[0] + finally: + for name in files: + (chart / name).unlink(missing_ok=True) + for path in (chart / "templates", chart): + if path.exists(): + path.rmdir() + + +def seed(port, root): + code, _ = request(port, "GET", "/apis/admissionregistration.k8s.io/v1/" + "validatingadmissionpolicies/kars-sre-binding-authority") + require(code == 404, "Legacy binding proof fixtures must precede admission policies") + reader = create(port, RBAC + "/clusterroles", historical_reader(root)) + custom = create(port, RBAC + "/clusterroles", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": CUSTOM}, + "rules": [{"apiGroups": [""], "resources": ["limitranges"], "verbs": ["get"]}]}) + bindings = [] + for role in (reader, custom): + bindings.append(create(port, RBAC + "/clusterrolebindings", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", + "metadata": {"name": role["metadata"]["name"]}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", + "name": role["metadata"]["name"]}, + "subjects": [LEGACY, SURVIVOR]})) + consumer = create(port, f"/apis/apps/v1/namespaces/{RUNTIME}/deployments", { + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "sre", "namespace": RUNTIME}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-reviewed-consumer"}}, + "template": {"metadata": {"labels": {"app": "e2e-reviewed-consumer"}}, + "spec": {"automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "never-executed", + "image": "registry.invalid/kars-admission-proof:never", "imagePullPolicy": "Never"}]}}}}) + return {"bindings": bindings, "roles": [reader, custom], "consumer": consumer} + + +def retirement_patch(binding, registration_uid): + require(binding["subjects"] == [LEGACY, SURVIVOR], "Unexpected reviewed binding subjects") + return {"metadata": {"uid": binding["metadata"]["uid"], + "resourceVersion": binding["metadata"]["resourceVersion"], + "annotations": {RETIRED: registration_uid}}, "subjects": [SURVIVOR]} + + +def authorization_category(code, body): + if not isinstance(body, dict): + return "unexpected-response" + if code == 403 and body.get("kind") == "Status" and body.get("reason") == "Forbidden": + message = body.get("message", "") + if isinstance(message, str) and "is attempting to grant RBAC permissions not currently held" in message: + return "rbac-permissions-not-held" + return "other-forbidden" + if code == 409 and body.get("kind") == "Status" and body.get("reason") == "Conflict": + return "cas-conflict" + if code == 422 and body.get("kind") == "Status" and body.get("reason") == "Invalid": + causes = body.get("details", {}).get("causes", []) + if any(cause.get("field") == "metadata.uid" and cause.get("reason") == "FieldValueInvalid" + and "field is immutable" in cause.get("message", "") for cause in causes): + return "immutable-uid" + if code == 200 and body.get("kind") == "ClusterRoleBinding": + return "accepted" + return "unexpected-response" + + +class ControllerAPI: + def __init__(self, root, port): + config_args = ["kubectl", "--context", CONTEXT, "config", "view", "--raw", "--minify", "-o"] + self.server = command("public-api-server", config_args + [ + "jsonpath={.clusters[0].cluster.server}"], root=root) + require(urlsplit(self.server).hostname in ("127.0.0.1", "localhost", "::1"), + "RBAC proof refuses a non-loopback API") + ca = command("public-api-ca", config_args + [ + "jsonpath={.clusters[0].cluster.certificate-authority-data}"], root=root) + context = ssl.create_default_context( + cadata=base64.b64decode(ca).decode()) + self.opener = build_opener(ProxyHandler({}), HTTPSHandler(context=context)) + account = get(port, f"/api/v1/namespaces/{SYSTEM}/serviceaccounts/kars-controller") + code, response = request(port, "POST", f"/api/v1/namespaces/{SYSTEM}/serviceaccounts/kars-controller/token", { + "apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest", + "spec": {"audiences": [], "expirationSeconds": 600}}) + require(code == 201 and response.get("status", {}).get("token"), + "Ephemeral controller-principal TokenRequest failed") + self.token = response["status"]["token"] + code, response = self.request("POST", "/apis/authentication.k8s.io/v1/selfsubjectreviews", { + "apiVersion": "authentication.k8s.io/v1", "kind": "SelfSubjectReview"}) + identity = response.get("status", {}).get("userInfo", {}) + require(code == 201 and identity.get("username") == CONTROLLER + and identity.get("uid") == account["metadata"]["uid"], + "Bearer-only proof principal does not match the actual controller ServiceAccount UID") + self.account = account + + def request(self, method, path, obj): + req = Request(self.server + path, data=json.dumps(obj).encode(), method=method, + headers={"Content-Type": "application/merge-patch+json" if method == "PATCH" else "application/json", + "Accept": "application/json", "Authorization": f"Bearer {self.token}"}) + try: + response = self.opener.open(req, timeout=15) + except HTTPError as error: + response = error + with response: + return response.code, json.loads(response.read(1024 * 1024)) + + def allowed(self, group, resource, verb, name=None): + attributes = {"group": group, "resource": resource, "verb": verb} + if name: + attributes["name"] = name + code, result = self.request("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", { + "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", + "spec": {"resourceAttributes": attributes}}) + require(code == 201 and isinstance(result.get("status", {}).get("allowed"), bool) + and not result["status"].get("evaluationError"), "Live controller authorization review failed") + return result["status"]["allowed"] + + +def observe_bind(api, expected): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", READER) is expected: + return + time.sleep(0.25) + raise AssertionError("Exact named bind permission did not converge") + + +def prove(root, port, state, objects, report): + from .bootstrap_probe import PATHS + source = create(port, f"/apis/kars.azure.com/v1alpha1/namespaces/{SYSTEM}/karssandboxes", { + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": "sre", "namespace": SYSTEM}, + "spec": {"runtime": {"kind": "BYO", "byo": {"contractVersion": "v1", + "image": "registry.invalid/kars-admission-proof:never", "command": ["/bin/true"]}}, + "inferenceRef": {"name": "sre-inference"}, "sandbox": {"isolation": "standard"}}}) + consumer_path = f"/apis/apps/v1/namespaces/{RUNTIME}/deployments/sre" + consumer = get(port, consumer_path) + spec = {"controller": {"namespace": {"name": SYSTEM, + "uid": get(port, f"/api/v1/namespaces/{SYSTEM}")["metadata"]["uid"]}, + "deployment": {"name": "kars-controller", "uid": get(port, + f"/apis/apps/v1/namespaces/{SYSTEM}/deployments/kars-controller")["metadata"]["uid"]}, "release": "kars"}, + "sandbox": {"namespace": SYSTEM, "name": "sre", "uid": source["metadata"]["uid"]}, + "runtimeNamespace": {"name": RUNTIME, "uid": get(port, f"/api/v1/namespaces/{RUNTIME}")["metadata"]["uid"]}, + "legacyBindings": [{"kind": "ClusterRoleBinding", **{k: binding["metadata"][k] + for k in ("name", "uid", "resourceVersion")}, "roleRef": binding["roleRef"], "subjects": binding["subjects"]} + for binding in state["bindings"]], + "legacyConsumer": {"namespace": RUNTIME, "name": "sre", **{k: consumer["metadata"][k] + for k in ("uid", "resourceVersion")}}} + registration = create(port, REGISTRATION.rsplit("/", 1)[0], { + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, "spec": spec}) + snapshots = {} + for obj in objects + state["roles"] + state["bindings"]: + if obj["kind"] not in ("ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding", + "ValidatingAdmissionPolicy", "ValidatingAdmissionPolicyBinding"): + continue + path = PATHS[obj["kind"]].format(namespace=obj["metadata"].get("namespace", SYSTEM)) + path += "/" + obj["metadata"]["name"] + current = get(port, path) + if obj["kind"] == "ValidatingAdmissionPolicy" and obj["metadata"]["name"].startswith("kars-sre-"): + require(current.get("status", {}).get("observedGeneration") == current["metadata"]["generation"] + and "typeChecking" in current["status"] + and not current["status"]["typeChecking"].get("expressionWarnings"), + "SRE policy is not observed and warning-free during RBAC proof") + if obj["kind"] == "ValidatingAdmissionPolicyBinding" and obj["metadata"]["name"].startswith("kars-sre-"): + require("Deny" in current["spec"]["validationActions"] + and current["spec"]["policyName"] == current["metadata"]["name"], + "SRE policy lacks its intended Deny binding during RBAC proof") + # Native CEL type-check status may evolve; specification/identity must not. + current.pop("status", None) + if obj["kind"] == "ValidatingAdmissionPolicy": + current["metadata"].pop("resourceVersion", None) + snapshots[path] = current + require({obj["metadata"]["name"] for obj in objects if obj["kind"] == "ValidatingAdmissionPolicy" + and obj["metadata"]["name"].startswith("kars-sre-")} == {"kars-sre-" + name for name in POLICIES}, + "RBAC experiment lacks the complete SRE policy set") + api = ControllerAPI(root, port) + facts = {"controllerBearerUidVerified": True, "workloadExecution": "not-attempted", + "legacyCommit": LEGACY_COMMIT, "observedWarningFreeDenyBoundSrePolicies": len(POLICIES), "cases": []} + report(facts) + require(api.allowed("rbac.authorization.k8s.io", "clusterrolebindings", "patch", READER), + "Controller lacks ordinary CRB PATCH; this is not the bind hypothesis") + require(api.allowed("kars.azure.com", "karssreregistrations", "use", "canonical"), + "Controller lacks canonical registrar use") + baseline_bind = api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", READER) + facts["authorization"] = {"patchReaderBinding": True, "useCanonical": True, "bindReader": baseline_bind, + "bindCustom": api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", CUSTOM), + "getCustomLimitRanges": api.allowed("", "limitranges", "get"), + "getReaderNodeMetrics": api.allowed("metrics.k8s.io", "nodes", "get"), + "getApiEndpoints": api.allowed("", "endpoints", "get", "kubernetes"), + "getOtherEndpoints": api.allowed("", "endpoints", "get", "unrelated"), + "listEndpoints": api.allowed("", "endpoints", "list")} + report(facts) + require(facts["authorization"]["getApiEndpoints"] + and not facts["authorization"]["getOtherEndpoints"] + and not facts["authorization"]["listEndpoints"], + "API endpoint discovery is missing or broader than the exact named GET") + require(baseline_bind, "Shipped controller lacks the exact named legacy reader bind permission") + facts["shippedReaderBindVerified"] = True + # The historical 403/200/403 experiment is retained in its immutable CI + # artifact. With the repaired chart, this extra grant is deliberately + # redundant: removing it must leave the shipped reader bind authorized. + facts["temporaryReaderBindGrant"] = "redundant-with-shipped-permission" + require(not facts["authorization"]["bindCustom"] and not facts["authorization"]["getCustomLimitRanges"], + "Custom fixture is not outside the controller's current authority") + + def unchanged(): + for path, before in snapshots.items(): + current = get(port, path) + current.pop("status", None) + if current["kind"] == "ValidatingAdmissionPolicy": + current["metadata"].pop("resourceVersion", None) + require(current == before, "RBAC experiment changed a policy, role or reviewed binding") + live = get(port, consumer_path) + require(all(live["metadata"][key] == consumer["metadata"][key] for key in ("uid", "resourceVersion")) + and live["spec"] == consumer["spec"], + "RBAC experiment changed or stopped the legacy consumer") + require(get(port, REGISTRATION) == registration, "Image-free experiment unexpectedly reconciled registration") + require(get(port, f"/api/v1/namespaces/{SYSTEM}/serviceaccounts/kars-controller") == api.account, + "Controller identity was replaced during the RBAC experiment") + for secret in ("sre-api-router-identity", "sre-api-agent"): + code, _ = request(port, "GET", f"/api/v1/namespaces/{RUNTIME}/secrets/{secret}") + require(code == 404, "RBAC dry-run experiment issued private material") + + def probe(binding, phase, expected, stale=None): + patch = retirement_patch(binding, registration["metadata"]["uid"]) + if stale: + patch["metadata"][stale] = "00000000-0000-0000-0000-000000000000" if stale == "uid" else "1" + code, response = api.request("PATCH", + RBAC + "/clusterrolebindings/" + binding["metadata"]["name"] + "?dryRun=All", patch) + category = authorization_category(code, response) + result = {"phase": phase, "role": binding["roleRef"]["name"], "httpStatus": code, "category": category} + facts["cases"].append(result) + report(facts) + require(code == expected and category == {200: "accepted", 403: "rbac-permissions-not-held", + 409: "cas-conflict", 422: "immutable-uid"}[expected], + "Retirement dry-run did not prove the intended API outcome") + if expected == 200: + require(response["metadata"]["uid"] == binding["metadata"]["uid"] + and response["metadata"].get("annotations", {}).get(RETIRED) == registration["metadata"]["uid"] + and response["roleRef"] == binding["roleRef"] and response["subjects"] == [SURVIVOR], + "Retirement dry-run did not preserve the exact unrelated subject and roleRef") + unchanged() + + reader, custom = state["bindings"] + probe(reader, "shipped-baseline", 200) + probe(custom, "baseline-custom", 403) + extra = [] + try: + role = create(port, RBAC + "/clusterroles", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", "metadata": {"name": EXTRA}, + "rules": [{"apiGroups": ["rbac.authorization.k8s.io"], "resources": ["clusterroles"], + "resourceNames": [READER], "verbs": ["bind"]}]}) + extra.append((RBAC + "/clusterroles/" + EXTRA, role)) + binding = create(port, RBAC + "/clusterrolebindings", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", "metadata": {"name": EXTRA}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": EXTRA}, + "subjects": [{"kind": "ServiceAccount", "name": "kars-controller", "namespace": SYSTEM}]}) + extra.append((RBAC + "/clusterrolebindings/" + EXTRA, binding)) + observe_bind(api, True) + require(not api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", CUSTOM) + and not api.allowed("", "limitranges", "get"), "Named reader bind widened custom-role authority") + probe(reader, "exact-reader-bind-only", 200) + probe(custom, "exact-reader-bind-only-custom", 403) + probe(reader, "wrong-uid", 422, "uid") + probe(reader, "stale-resource-version", 409, "resourceVersion") + facts["allReviewedDryRunsAuthorized"] = False + facts["noRetirementsApplied"] = True + finally: + for path, obj in reversed(extra): + code, _ = request(port, "DELETE", path, { + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {k: obj["metadata"][k] for k in ("uid", "resourceVersion")}}) + require(code in (200, 202), "Exact named bind fixture cleanup failed") + observe_bind(api, baseline_bind) + probe(reader, "temporary-bind-removed-shipped-bind-remains", 200) + facts["unchangedPoliciesRolesBindingsConsumer"] = True + facts["testBindGrantRemoved"] = True + report(facts) diff --git a/tests/e2e/sre_authority/binding_probe_test.py b/tests/e2e/sre_authority/binding_probe_test.py new file mode 100644 index 000000000..ff0527b7b --- /dev/null +++ b/tests/e2e/sre_authority/binding_probe_test.py @@ -0,0 +1,132 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure privacy/request-shape checks; the A/B result requires actual Kind.""" + +import base64 +import copy +import json +from pathlib import Path +import types +import unittest +from unittest.mock import patch + +from sre_authority.binding_probe import ( + CONTROLLER, LEGACY, RETIRED, SURVIVOR, ControllerAPI, authorization_category, retirement_patch, +) +from sre_authority.bootstrap_probe import failure_site + + +class BindingProbeTests(unittest.TestCase): + def test_proof_failure_reports_only_public_source_coordinate_not_values(self): + try: + retirement_patch({"subjects": ["private-body-do-not-log"]}, "private-token-do-not-log") + except AssertionError as error: + facts = failure_site(error) + self.assertEqual(facts["category"], "AssertionError") + self.assertEqual(facts["source"], "binding_probe.py") + self.assertGreater(facts["line"], 0) + self.assertNotIn("private", json.dumps(facts)) + + def test_patch_is_exact_subtractive_review_with_uid_rv_and_registration_fence(self): + binding = {"metadata": {"uid": "binding-uid", "resourceVersion": "123"}, + "subjects": [LEGACY, SURVIVOR]} + original = copy.deepcopy(binding) + result = retirement_patch(binding, "registration-uid") + self.assertEqual(result, {"metadata": {"uid": "binding-uid", "resourceVersion": "123", + "annotations": {RETIRED: "registration-uid"}}, "subjects": [SURVIVOR]}) + self.assertEqual(binding, original) + for subjects in ([SURVIVOR], [LEGACY], [LEGACY, SURVIVOR, {"kind": "User", "name": "unreviewed"}]): + with self.assertRaises(AssertionError): + retirement_patch({**binding, "subjects": subjects}, "registration-uid") + + def test_arbitrary_forbidden_or_failure_is_not_escalation_proof(self): + denied = {"kind": "Status", "reason": "Forbidden", + "message": "user is attempting to grant RBAC permissions not currently held: private-not-for-logs"} + self.assertEqual(authorization_category(403, denied), "rbac-permissions-not-held") + for message in ("cannot patch clusterrolebindings", "kars-sre-binding-authority denied", None, {}, ""): + self.assertEqual(authorization_category(403, {**denied, "message": message}), "other-forbidden") + for code in (200, 201, 401, 404, 409, 422, 500): + self.assertEqual(authorization_category(code, denied), "unexpected-response") + self.assertEqual(authorization_category(200, {"kind": "ClusterRoleBinding"}), "accepted") + self.assertEqual(authorization_category(409, {"kind": "Status", "reason": "Conflict"}), "cas-conflict") + self.assertEqual(authorization_category(403, None), "unexpected-response") + self.assertNotIn("private-not-for-logs", authorization_category(403, denied)) + + def test_uid_rejection_is_specific_builtin_validation_not_arbitrary_422(self): + cause = {"field": "metadata.uid", "reason": "FieldValueInvalid", "message": "field is immutable"} + response = {"kind": "Status", "reason": "Invalid", "details": {"causes": [cause]}} + self.assertEqual(authorization_category(422, response), "immutable-uid") + for changed in ({"field": "subjects"}, {"reason": "Unexpected"}, {"message": "different error"}): + bad = {**response, "details": {"causes": [{**cause, **changed}]}} + self.assertEqual(authorization_category(422, bad), "unexpected-response") + self.assertEqual(authorization_category(403, response), "unexpected-response") + + def test_controller_token_is_memory_only_no_admin_certificate_and_real_uid_is_required(self): + account = {"metadata": {"uid": "actual-controller-uid"}} + captured = [] + def api_request(_self, method, path, obj): + captured.append((method, path, obj)) + return 201, {"status": {"userInfo": {"username": CONTROLLER, "uid": "actual-controller-uid"}}} + def config(stage, *_args, **_kwargs): + return "https://127.0.0.1:6443" if stage == "public-api-server" else base64.b64encode(b"public CA").decode() + context = types.SimpleNamespace() + with patch("sre_authority.binding_probe.command", side_effect=config), \ + patch("sre_authority.binding_probe.get", return_value=account), \ + patch("sre_authority.binding_probe.request", return_value=(201, {"status": {"token": "unit-only-token"}})), \ + patch("sre_authority.binding_probe.ssl.create_default_context", return_value=context) as ssl_context, \ + patch("sre_authority.binding_probe.HTTPSHandler") as https, \ + patch("sre_authority.binding_probe.build_opener"), \ + patch.object(ControllerAPI, "request", api_request): + client = ControllerAPI(Path("."), 1) + self.assertEqual(client.account, account) + ssl_context.assert_called_once_with(cadata="public CA") + https.assert_called_once_with(context=context) + self.assertTrue(captured[0][1].endswith("/selfsubjectreviews")) + with patch.object(ControllerAPI, "request", return_value=(201, { + "status": {"userInfo": {"username": CONTROLLER, "uid": "replacement"}}})): + with self.assertRaises(AssertionError): + ControllerAPI(Path("."), 1) + + def test_controller_dry_run_has_real_bearer_and_merge_patch_without_impersonation(self): + class Response: + code = 403 + def __enter__(self): + return self + def __exit__(self, *_args): + pass + def read(self, _limit): + return b'{"kind":"Status","reason":"Forbidden"}' + sent = [] + def open_request(request, **_kwargs): + sent.append(request) + return Response() + client = ControllerAPI.__new__(ControllerAPI) + client.server, client.token = "https://127.0.0.1:6443", "unit-only-token" + client.opener = types.SimpleNamespace(open=open_request) + code, _ = client.request("PATCH", "/fixture?dryRun=All", {"subjects": [SURVIVOR]}) + self.assertEqual(code, 403) + self.assertEqual(sent[0].get_header("Authorization"), "Bearer unit-only-token") + self.assertEqual(sent[0].get_header("Content-type"), "application/merge-patch+json") + self.assertFalse(any(key.lower().startswith("impersonate") for key in sent[0].headers)) + self.assertEqual(json.loads(sent[0].data), {"subjects": [SURVIVOR]}) + + def test_fast_hosted_gate_keeps_prior_admission_cases_and_adds_real_bind_experiment(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + fast = workflow.split(" sre-crd-schema:", 1)[1].split(" helm-lint:", 1)[0] + self.assertIn("sre_authority.bootstrap_probe --retirement-bind-proof", fast) + self.assertIn("sre_authority.binding_probe_test", fast) + source = (root / "tests/e2e/sre_authority/bootstrap_probe.py").read_text() + self.assertLess(source.index("cases = admission_cases"), source.index("prove(root, port, state")) + probe = (root / "tests/e2e/sre_authority/binding_probe.py").read_text() + self.assertIn('"resourceNames": [READER], "verbs": ["bind"]', probe) + self.assertNotIn('"verbs": ["escalate"]', probe) + self.assertNotIn('"resourceNames": ["*"]', probe) + self.assertIn('require(baseline_bind, "Shipped controller lacks', probe) + self.assertIn('"temporary-bind-removed-shipped-bind-remains", 200', probe) + self.assertIn('"redundant-with-shipped-permission"', probe) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py new file mode 100644 index 000000000..cff0b615e --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -0,0 +1,292 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Fixed public admission cases for the image-free schema candidate.""" + +import copy +import json +import time +from urllib.error import HTTPError +from urllib.request import Request, build_opener, ProxyHandler + +from .bootstrap_diagnostics import api_result, object_status +from .bootstrap_probe import upsert +from .registration_schema import request + +USER = "system:serviceaccount:e2e-sre-bootstrap:tenant" +DEPLOYMENT_CONTROLLER = "system:serviceaccount:kube-system:deployment-controller" + + +def as_tenant(port, path, obj, *, user=USER, method="POST"): + req = Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(obj).encode(), method=method, + headers={"Content-Type": "application/json", "Accept": "application/json", + "Impersonate-User": user}) + try: + response = build_opener(ProxyHandler({})).open(req, timeout=15) + except HTTPError as error: + response = error + with response: + return response.code, json.loads(response.read(1024 * 1024)) + + +def admission_cases(port, policies): + code, _ = request(port, "POST", "/api/v1/namespaces", { + "apiVersion": "v1", "kind": "Namespace", "metadata": {"name": "e2e-sre-bootstrap"}}) + if code not in (201, 409): + raise RuntimeError("Disposable admission principal namespace unavailable") + setup = [ + {"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "tenant", "namespace": "e2e-sre-bootstrap"}}, + {"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "sandbox", "namespace": "kars-sre"}}, + {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", + "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, + "rules": [{"apiGroups": [""], "resources": ["pods"], "verbs": ["create"]}, + {"apiGroups": [""], "resources": ["replicationcontrollers"], "verbs": ["create", "update"]}, + {"apiGroups": ["apps"], "resources": ["deployments", "replicasets"], "verbs": ["create", "update"]}, + {"apiGroups": ["kars.azure.com"], "resources": ["karssreactions"], "verbs": ["create"]}]}, + {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "e2e-bootstrap-probe"}, + "subjects": [{"kind": "ServiceAccount", "name": "tenant", "namespace": "e2e-sre-bootstrap"}]}, + ] + for obj in setup: + if not upsert(port, obj, policies).get("accepted"): + raise RuntimeError("Disposable namespaced admission principal setup failed") + pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "e2e-ordinary", "namespace": "kars-sre"}, + "spec": {"serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]}} + cases = [("ordinary-tenant-pod", "/api/v1/namespaces/kars-sre/pods", pod, 201, None)] + private = copy.deepcopy(pod) + private["spec"]["volumes"] = [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}] + cases.append(("private-volume", "/api/v1/namespaces/kars-sre/pods", private, 403, "kars-sre-private-mounts")) + env = copy.deepcopy(pod) + env["spec"]["containers"][0]["envFrom"] = [{"secretRef": {"name": "sre-api-router-identity"}}] + cases.append(("private-env", "/api/v1/namespaces/kars-sre/pods", env, 403, "kars-sre-private-mounts")) + for kind, plural in (("Deployment", "deployments"), ("ReplicaSet", "replicasets"), + ("ReplicationController", "replicationcontrollers")): + for is_private in (False, True): + replication_controller = kind == "ReplicationController" + obj = {"apiVersion": "v1" if replication_controller else "apps/v1", "kind": kind, + "metadata": {"name": "e2e-template", "namespace": "kars-sre"}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-probe"}}, + "template": {"metadata": {"labels": {"app": "e2e-probe"}}, + "spec": copy.deepcopy((private if is_private else pod)["spec"])}}} + if replication_controller: + obj["spec"]["selector"] = {"app": "e2e-probe"} + obj["spec"]["replicas"] = 0 + cases.append((f"{kind}-{'private' if is_private else 'ordinary'}", + f"{'/api/v1' if replication_controller else '/apis/apps/v1'}/namespaces/kars-sre/{plural}", obj, + 403 if is_private else 201, "kars-sre-private-workloads" if is_private else None)) + if replication_controller and not is_private: + no_template = copy.deepcopy(obj) + no_template["spec"].pop("template") + cases.append(("ReplicationController-no-template", + "/api/v1/namespaces/kars-sre/replicationcontrollers", no_template, 422, None)) + private_account = copy.deepcopy(obj) + private_account["spec"]["template"]["spec"]["serviceAccountName"] = "sre-api-router" + cases.append(("ReplicationController-private-account", + "/api/v1/namespaces/kars-sre/replicationcontrollers", + private_account, 403, "kars-sre-private-workloads")) + params = {"namespace": "example", "name": "demo", "replicas": 1, + "nested": {"array": [True, None, 1, "text"], "object": {"key": "value"}}} + action = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSREAction", + "metadata": {"name": "e2e-proposal", "namespace": "kars-sre"}, + "spec": {"action": {"type": "ScaleDeployment", "params": params}, + "approval": {"state": "Pending"}, "ttlMinutes": 5}} + cases.append(("pending-json-params-preserved", "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions", + action, 201, None)) + approved = copy.deepcopy(action) + approved["spec"]["approval"]["state"] = "Approved" + cases.append(("preapproved-action-denied", "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions", + approved, 403, "kars-sre-pending-proposals")) + reports = [] + for name, path, obj, expected, policy in cases: + code, response = as_tenant(port, path + "?dryRun=All", obj) + result = api_result(code, response, policies) + result.update({"case": name, "expectedStatus": expected, "matched": code == expected + and (policy is None or policy in result.get("policies", []))}) + if name == "pending-json-params-preserved": + result["paramsPreserved"] = response.get("spec", {}).get("action", {}).get("params") == params + result["matched"] = result["matched"] and result["paramsPreserved"] + if name == "ReplicationController-no-template": + result["nativeTemplateRequired"] = response.get("reason") == "Invalid" and any( + cause.get("field") == "spec.template" and cause.get("reason") == "FieldValueRequired" + for cause in response.get("details", {}).get("causes", [])) + result["matched"] = result["matched"] and result["nativeTemplateRequired"] + reports.append(result) + return reports + + +def deployment_controller_cases(port, policies): + code, account = request(port, "GET", "/api/v1/namespaces/kube-system/serviceaccounts/deployment-controller") + if code != 200 or not account.get("metadata", {}).get("uid"): + raise RuntimeError("The actual Kubernetes Deployment controller ServiceAccount is absent") + authorization = {} + for label, group, resource, verb, name in ( + ("createReplicaSetsClusterWide", "apps", "replicasets", "create", None), + ("createPodsClusterWide", "", "pods", "create", None), + ("useRegistrar", "kars.azure.com", "karssreregistrations", "use", "canonical"), + ): + attributes = {"group": group, "resource": resource, "verb": verb} + if name: + attributes["name"] = name + code, body = request(port, "POST", "/apis/authorization.k8s.io/v1/subjectaccessreviews", { + "apiVersion": "authorization.k8s.io/v1", "kind": "SubjectAccessReview", + "spec": {"user": DEPLOYMENT_CONTROLLER, "resourceAttributes": attributes}, + }) + if code != 201 or not isinstance(body.get("status", {}).get("allowed"), bool): + raise RuntimeError("Deployment controller authorization proof failed") + authorization[label] = body["status"]["allowed"] + reports = [] + for private in (False, True): + pod = {"serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]} + if private: + pod["volumes"] = [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}] + obj = {"apiVersion": "apps/v1", "kind": "ReplicaSet", + "metadata": {"name": "e2e-deployment-controller", "namespace": "kars-sre"}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-controller"}}, + "template": {"metadata": {"labels": {"app": "e2e-controller"}}, "spec": pod}}} + code, body = as_tenant(port, "/apis/apps/v1/namespaces/kars-sre/replicasets?dryRun=All", + obj, user=DEPLOYMENT_CONTROLLER) + result = api_result(code, body, policies) + result.update({"case": f"deployment-controller-{'private' if private else 'ordinary'}-replicaset", + "expectedStatus": 201, "matched": code == 201, + "actualControllerAccountExists": True, "authorization": authorization, + "identityMode": "admin-impersonation-of-built-in-controller"}) + reports.append(result) + return reports + + +def private_controller_chain(port, policies, report): + namespace, name = "kars-sre", "e2e-private-controller-chain" + selector = {"app": name} + deployment = {"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": name, "namespace": namespace}, + "spec": {"replicas": 1, "selector": {"matchLabels": selector}, + "template": {"metadata": {"labels": selector}, "spec": { + "serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never", + "volumeMounts": [{"name": "private", "mountPath": "/private", "readOnly": True}]}], + "volumes": [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}], + }}}} + path = f"/apis/apps/v1/namespaces/{namespace}/deployments" + code, created = request(port, "POST", path, deployment) + if code != 201 or not created.get("metadata", {}).get("uid"): + report({"deploymentCreate": api_result(code, created, policies)}) + raise RuntimeError("Registrar-authorized private Deployment CREATE failed") + uid = created["metadata"]["uid"] + deadline = time.monotonic() + 45 + snapshot = {} + while time.monotonic() < deadline: + code, current = request(port, "GET", f"{path}/{name}") + if code != 200 or current.get("metadata", {}).get("uid") != uid: + raise RuntimeError("Private controller-chain Deployment disappeared or was replaced") + code, replicasets = request(port, "GET", + f"/apis/apps/v1/namespaces/{namespace}/replicasets?labelSelector=app%3D{name}") + if code != 200 or not isinstance(replicasets.get("items"), list): + raise RuntimeError("Private controller-chain ReplicaSet inspection failed") + owned = [obj for obj in replicasets["items"] if any( + owner.get("uid") == uid and owner.get("controller") is True + for owner in obj.get("metadata", {}).get("ownerReferences", []))] + owners = {obj["metadata"]["uid"] for obj in owned} + code, pods = request(port, "GET", f"/api/v1/namespaces/{namespace}/pods?labelSelector=app%3D{name}") + if code != 200 or not isinstance(pods.get("items"), list): + raise RuntimeError("Private controller-chain Pod inspection failed") + children = [obj for obj in pods["items"] if any( + owner.get("uid") in owners and owner.get("controller") is True + for owner in obj.get("metadata", {}).get("ownerReferences", []))] + snapshot = {"deployment": object_status(current, policies), + "replicaSets": [object_status(dict(obj, kind="ReplicaSet"), policies) for obj in owned], + "pods": [object_status(dict(obj, kind="Pod"), policies) for obj in children], + "privateMountPreserved": False, "noWorkloadExecution": False} + if children: + snapshot["privateMountPreserved"] = all( + obj["spec"].get("volumes") and any( + volume.get("secret", {}).get("secretName") == "sre-api-router-identity" + for volume in obj["spec"]["volumes"]) + and any(mount.get("name") == "private" and mount.get("mountPath") == "/private" + and mount.get("readOnly") is True + for container in obj["spec"].get("containers", []) + for mount in container.get("volumeMounts", [])) + for obj in children) + snapshot["noWorkloadExecution"] = all( + obj["spec"].get("schedulerName") == "kars-e2e-admission-never-schedule" + and not obj["spec"].get("nodeName") and not obj.get("status", {}).get("containerStatuses") + for obj in children) + report(snapshot) + if not snapshot["privateMountPreserved"] or not snapshot["noWorkloadExecution"]: + raise RuntimeError("Private controller-chain proof changed its protected template or executed a workload") + return created + time.sleep(0.5) + report(snapshot) + raise RuntimeError("Actual private Deployment/ReplicaSet controllers did not create the admission-only Pod") + + +def namespace_cleanup_cases(port, policies, owned_consumer=None): + path = "/apis/apps/v1/namespaces/kars-sre/deployments/sre" + code, current = request(port, "GET", path) + if owned_consumer is not None: + if (code != 200 or current.get("metadata", {}).get("uid") != owned_consumer["metadata"]["uid"] + or current.get("spec") != owned_consumer.get("spec")): + raise RuntimeError("Earlier owned cleanup fixture changed; no adoption permitted") + created = current + elif code != 404: + raise RuntimeError("Namespace cleanup proof refuses an existing canonical Deployment") + obj = {"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": "sre", "namespace": "kars-sre"}, + "spec": {"replicas": 0, "selector": {"matchLabels": {"app": "e2e-retired-consumer"}}, + "template": {"metadata": {"labels": {"app": "e2e-retired-consumer"}}, "spec": { + "automountServiceAccountToken": False, "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]}}}} + if owned_consumer is None: + code, created = request(port, "POST", path.rsplit("/", 1)[0], obj) + if code != 201 or not created.get("metadata", {}).get("uid"): + raise RuntimeError("Canonical no-execution cleanup fixture CREATE failed") + uid = created["metadata"]["uid"] + reports = [] + primary_failure = False + try: + for account, namespace, expected in (("namespace-controller", "kube-system", 403), + ("kars-controller", "kars-system", 200)): + principal = f"system:serviceaccount:{namespace}:{account}" + code, sa = request(port, "GET", f"/api/v1/namespaces/{namespace}/serviceaccounts/{account}") + if code != 200 or not sa.get("metadata", {}).get("uid"): + raise RuntimeError("Cleanup proof principal does not exist") + code, current = request(port, "GET", path) + if code != 200 or current.get("metadata", {}).get("uid") != uid: + raise RuntimeError("Cleanup proof Deployment was replaced before its dry-run") + code, response = as_tenant(port, path, { + "apiVersion": "v1", "kind": "DeleteOptions", "dryRun": ["All"], + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, + user=principal, method="DELETE") + result = api_result(code, response, policies) + result.update({"case": f"{account}-canonical-consumer-delete", "expectedStatus": expected, + "matched": code == expected and (code != 403 + or "kars-sre-consumer-authority" in result.get("policies", []))}) + reports.append(result) + return reports + except BaseException: + primary_failure = True + raise + finally: + try: + code, current = request(port, "GET", path) + if code != 200 or current.get("metadata", {}).get("uid") != uid: + raise RuntimeError("Cleanup proof Deployment identity changed") + code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}) + if code not in (200, 202): + raise RuntimeError("Cleanup proof could not remove its owned Deployment") + except Exception: + if not primary_failure: + raise + print("SRE-DIAG owned canonical cleanup fixture removal unavailable", flush=True) diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py new file mode 100644 index 000000000..3d615c049 --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -0,0 +1,414 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Allowlisted public admission evidence; never dump Pod specs, argv or bodies.""" + +import json +import re +import subprocess + +REASONS = { + "Forbidden", "Invalid", "InternalError", "BadRequest", "NotFound", "AlreadyExists", + "Unauthorized", "Conflict", "ServiceUnavailable", "FailedCreate", "ReplicaFailure", + "MinimumReplicasUnavailable", "MinimumReplicasAvailable", "NewReplicaSetCreated", + "FoundNewReplicaSet", "ReplicaSetUpdated", "NewReplicaSetAvailable", + "ReplicaSetCreateError", "ProgressDeadlineExceeded", "FailedScheduling", + "FailedMount", "Failed", "SuccessfulCreate", "ScalingReplicaSet", +} +FIELDS = { + "namespace", "name", "labels", "annotations", "spec", "status", "subjects", + "enabled", "serviceAccountName", "containers", "initContainers", "ephemeralContainers", + "securityContext", "operation", "subResource", "container", "kind", "apiVersion", +} + + +def probe_command_result(code, output): + category = "unclassified" + if code == 0: + category = "succeeded" + elif "executable file not found" in output: + category = "executable-not-found" + elif code == 1 and output.strip() in ("", "command terminated with exit code 1"): + category = "probe-not-ready" + return {"exitCode": code, "category": category} + + +def router_readiness_facts(text): + facts = [] + stages = ("registration", "namespace", "source", "service-account", "privacy-review", "credential-metadata", + "enabled", "loaded", "bound", "serving", "tcp-accepted", "tls-accepted", "tls-rejected", + "tls-timeout", "readiness-entered", "authority-entered") + steps = ("token", "request", "headers", "complete") + categories = ("authority-transport", "authority-denied", "registration-stale", "namespace-claim", + "source-identity", "service-account", "privacy-transport", "privacy-denied", + "privacy-not-denied", "metadata-transport", "metadata-denied", "legacy-alias", + "credential-expired", "unclassified") + messages = ("SRE authority transport failure", "SRE authority request denied", + "SRE readiness authority rejected", "SRE readiness authority slow", + "SRE transport progress", "SRE authority progress") + for line in text.splitlines()[-150:]: + line = re.sub(r"\x1b\[[0-9;]*m", "", line) + try: + event = json.loads(line) + except ValueError: + event = None + if isinstance(event, dict): + fields = event.get("fields", event) + if not isinstance(fields, dict) or fields.get("message") not in messages: + continue + value = {} + for key, allowed in (("stage", stages), ("category", categories), ("step", steps)): + if fields.get(key) in allowed: + value[key] = fields[key] + for key in ("timed_out", "connect_error", "authorized"): + if isinstance(fields.get(key), bool): + value[key] = fields[key] + if type(fields.get("http_status")) is int and 100 <= fields["http_status"] <= 599: + value["httpStatus"] = fields["http_status"] + if type(fields.get("elapsed_seconds")) is int and 0 <= fields["elapsed_seconds"] <= 999: + value["elapsedSeconds"] = fields["elapsed_seconds"] + if value and value not in facts: + facts.append(value) + continue + if not any(message in line for message in messages): + continue + value = {} + for key, allowed in (("stage", stages), ("category", categories), ("step", steps)): + match = re.search(rf'\b{key}="?({"|".join(allowed)})"?(?:\s|$)', line) + if match: + value[key] = match[1] + for key in ("timed_out", "connect_error", "authorized"): + match = re.search(rf"\b{key}=(true|false)(?:\s|$)", line) + if match: + value[key] = match[1] == "true" + status = re.search(r"\bhttp_status=([1-5][0-9]{2})(?:\s|$)", line) + if status: + value["httpStatus"] = int(status[1]) + elapsed = re.search(r"\belapsed_seconds=([0-9]{1,3})(?:\s|$)", line) + if elapsed: + value["elapsedSeconds"] = int(elapsed[1]) + if value and value not in facts: + facts.append(value) + return facts[:32] + + +def router_log_summary(text, root): + paths = [ + root / "inference-router/src/main.rs", + root / "inference-router/src/routes/mod.rs", + root / "inference-router/src/sre_proxy/mod.rs", + root / "inference-router/src/sre_proxy/backend.rs", + ] + sources = [(path, path.read_text().splitlines()) for path in paths] + summary = {"lines": len(text.splitlines()), "jsonEvents": 0, "sourceSites": []} + for line in text.splitlines()[-150:]: + try: + obj = json.loads(line) + except ValueError: + continue + if not isinstance(obj, dict): + continue + summary["jsonEvents"] += 1 + fields = obj.get("fields", obj) + if not isinstance(fields, dict) or not isinstance(fields.get("message"), str): + continue + literal = json.dumps(fields["message"], ensure_ascii=False) + for path, lines in sources: + match = next((number for number, source in enumerate(lines, 1) if literal in source), None) + if match: + site = {"source": str(path.relative_to(root)), "line": match} + if site not in summary["sourceSites"]: + summary["sourceSites"].append(site) + break + summary["sourceSites"] = summary["sourceSites"][:32] + return summary + + +def firewall_summary(text): + result = {"policies": [], "rules": []} + table = None + for line in text.splitlines(): + if line.startswith("*"): + table = line[1:] if line[1:] in ("filter", "nat", "raw", "mangle", "security") else None + if not table: + continue + policy = re.fullmatch(r":(INPUT|OUTPUT|FORWARD) (ACCEPT|DROP) \[([0-9]+):[0-9]+\]", line) + if policy: + result["policies"].append({"table": table, "chain": policy[1], + "policy": policy[2], "packets": int(policy[3])}) + match = re.match(r"(?:\[([0-9]+):[0-9]+\] )?-A ([A-Za-z0-9_-]+) ", line) + if not match: + continue + owner = re.search(r"(! )?--uid-owner (1000|1001)(?:\s|$)", line) + action = re.search(r" -j (ACCEPT|DROP|REJECT|REDIRECT|RETURN)(?:\s|$)", line) + result["rules"].append({ + "table": table, "chain": match[2] if match[2] in ("INPUT", "OUTPUT", "FORWARD") else "custom", + "packets": int(match[1]) if match[1] else None, + "action": action[1] if action else "jump-or-other", + "owner": int(owner[2]) if owner else None, + "ownerNegated": bool(owner and owner[1]), + "loopback": bool(re.search(r"(?: -[io] lo)(?:\s|$)", line)), + "established": "--ctstate" in line and "ESTABLISHED" in line, + }) + result["rules"] = result["rules"][:40] + return result + + +def exception_summary(error, root): + result = {"category": type(error).__name__} + frame = error.__traceback__ + while frame: + source = str(frame.tb_frame.f_code.co_filename) + prefix = str(root / "tests/e2e/sre_authority") + "/" + if source.startswith(prefix): + result["callSite"] = {"source": source[len(str(root)) + 1:], + "function": frame.tb_frame.f_code.co_name, "line": frame.tb_lineno} + frame = frame.tb_next + response = getattr(error, "response", None) + status = getattr(response, "status_code", None) + if type(status) is not int or not 100 <= status <= 599: + return result + result["httpStatus"] = status + try: + body = response.json() + except (ValueError, TypeError): + return result + result.update(response_summary(status, body, root)) + return result + + +def response_summary(status, body, root): + result = {"httpStatus": status} + if not isinstance(body, dict) or body.get("kind") != "Status": + return result + reason = body.get("reason") + if isinstance(reason, str) and (reason in REASONS or reason == "SREProxyDenied"): + result["reason"] = reason + message = body.get("message") + if not isinstance(message, str) or not 0 < len(message) <= 4096: + return result + literal = json.dumps(message, ensure_ascii=False) + for name in ("mod.rs", "policy.rs", "backend.rs"): + path = root / "inference-router/src/sre_proxy" / name + for number, source in enumerate(path.read_text().splitlines(), 1): + if literal in source: + result["responseSite"] = {"source": str(path.relative_to(root)), "line": number} + return result + return result + + +def identifier(value): + return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None + + +def failure_facts(message, policies): + if not isinstance(message, str): + return {} + message = message[:65536] + facts = { + "policies": sorted(name for name in policies if name in message), + "validationMessages": [], + "categories": [label for label, needle in [ + ("compilation", "compilation"), ("evaluation", "evaluation"), + ("no-such-key", "no such key"), ("undefined-field", "undefined field"), + ("undeclared-reference", "undeclared reference"), ("forbidden", "forbidden"), + ("not-found", "not found"), ("quota", "quota"), ("type-checking", "type checking"), + ] if needle in message.lower()], + "missingFields": sorted(set(re.findall(r"no such key: ([A-Za-z_][A-Za-z0-9_]*)", message)) & FIELDS), + } + for name, policy in policies.items(): + for index, validation in enumerate(policy.get("spec", {}).get("validations", [])): + known = validation.get("message") + if isinstance(known, str) and known in message: + facts["validationMessages"].append({"policy": name, "index": index, "message": known}) + facts["serviceAccountMissing"] = 'serviceaccount "kars-controller" not found' in message.lower() + return facts + + +def api_result(code, body, policies): + report = {"httpStatus": code} + if isinstance(body, dict) and body.get("kind") == "Status": + reason = body.get("reason") + report["reason"] = reason if isinstance(reason, str) and reason in REASONS else "unclassified" + report.update(failure_facts(body.get("message"), policies)) + details = body.get("details", {}) + if (code == 422 and reason == "Invalid" and isinstance(details, dict) + and details.get("group") == "admissionregistration.k8s.io" + and details.get("kind") == "ValidatingAdmissionPolicy" + and isinstance(details.get("name"), str) + and details.get("name") in policies): + causes = [] + supplied = details.get("causes", []) + if not isinstance(supplied, list): + return report + for cause in supplied[:32]: + if not isinstance(cause, dict): + continue + field, message = cause.get("field"), cause.get("message") + location = re.fullmatch( + r"spec\.(matchConditions|variables|validations)\[(\d+)\]\.expression", field + ) if isinstance(field, str) and len(field) <= 128 else None + if not location: + continue + if not isinstance(message, str): + continue + allowed = FIELDS | {"metadata", "request", "object", "oldObject", "orValue", "has", "exists"} + tokens = sorted(set(re.findall( + r"(?:undefined field|undeclared reference to) ['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]", + message)) & allowed) + entry = {"field": field, "knownTokens": tokens, + "categories": [category for category, needle in [ + ("overload", "matching overload"), ("syntax", "Syntax error"), + ("undefined-field", "undefined field"), ("undeclared-reference", "undeclared reference"), + ("optional", "optional"), ("cost", "cost"), + ] if needle.lower() in message.lower()]} + definitions = policies[details["name"]].get("spec", {}).get(location[1], []) + index = int(location[2]) + expression = definitions[index].get("expression", "") if index < len(definitions) else "" + vocabulary = allowed | set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", expression)) | { + "error", "input", "expression", "must", "evaluate", "evaluates", "return", "returns", + "type", "bool", "boolean", "string", "int", "list", "map", "dyn", "invalid", "argument", + "macro", "expected", "found", "no", "matching", "overload", "applied", "to", "in", + "not", "a", "an", "is", "of", "undeclared", "reference", "undefined", "field", + "mismatched", "extraneous", "syntax", "token", "reserved", "identifier", "unsupported", + "supported", "allowed", "size", "exceeds", "maximum", "limit", "cost", "compilation", + } + headline = message.partition("compilation failed:")[2].splitlines() + if headline: + entry["compilerDescription"] = re.sub( + r"[A-Za-z0-9_]+", + lambda match: match[0] if match[0] in vocabulary or match[0].lower() in vocabulary else "[redacted]", + re.sub(r"[\x00-\x1f\x7f]", "?", headline[0].strip())[:512], + ) + causes.append(entry) + report["compilationCauses"] = causes + return report + + +def object_status(obj, policies): + metadata, status = obj.get("metadata", {}), obj.get("status", {}) + report = {"kind": obj.get("kind"), "name": identifier(metadata.get("name")), + "namespace": identifier(metadata.get("namespace")), "uid": identifier(metadata.get("uid"))} + for field in ("replicas", "readyReplicas", "availableReplicas", "observedGeneration"): + if isinstance(status.get(field), int): + report[field] = status[field] + conditions = [] + for condition in status.get("conditions", [])[:16]: + if not isinstance(condition, dict): + continue + entry = {"type": identifier(condition.get("type")), "status": condition.get("status") + if condition.get("status") in ("True", "False", "Unknown") else None, + "reason": condition.get("reason") if condition.get("reason") in REASONS else "unclassified"} + entry.update(failure_facts(condition.get("message"), policies)) + conditions.append(entry) + report["conditions"] = conditions + return report + + +def policy_status(obj, expected): + name, status = obj.get("metadata", {}).get("name"), obj.get("status", {}) + if name not in expected: + raise AssertionError("Only rendered public policies may publish type-check diagnostics") + warnings = [] + for warning in status.get("typeChecking", {}).get("expressionWarnings", [])[:32]: + field, text = warning.get("fieldRef"), warning.get("warning") + if (isinstance(field, str) and re.fullmatch( + r"spec\.(matchConditions|validations|variables)\[\d+\]\.expression", field) + and isinstance(text, str)): + warnings.append({"fieldRef": field, "warning": re.sub( + r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "?", text)[:16384]}) + return {"name": name, "generation": obj.get("metadata", {}).get("generation"), + "observedGeneration": status.get("observedGeneration"), + "typeChecked": "typeChecking" in status, "warnings": warnings} + + +def control_plane_status(obj): + containers = [] + for container in obj.get("status", {}).get("containerStatuses", [])[:4]: + state = container.get("state", {}) + previous = container.get("lastState", {}).get("terminated", {}) + containers.append({"name": identifier(container.get("name")), + "ready": container.get("ready") is True, "restartCount": container.get("restartCount"), + "state": next((name for name in ("running", "waiting", "terminated") if name in state), "unknown"), + "previousExitCode": previous.get("exitCode"), + "previousReason": previous.get("reason") if previous.get("reason") in ("Error", "Completed", "OOMKilled") else None}) + return {"name": identifier(obj.get("metadata", {}).get("name")), "containers": containers} + + +def public_stack_facts(text): + """Only fixed panic classifications and public Go frame names, never log text/args.""" + facts = {"panicCategories": [], "publicFrames": []} + for line in text.splitlines()[-240:]: + line = line.strip() + if line.startswith("panic:") or line.startswith("fatal error:"): + category = next((name for name, phrase in [ + ("nil-pointer", "nil pointer dereference"), + ("index-out-of-range", "index out of range"), + ("interface-conversion", "interface conversion"), + ("concurrent-map-write", "concurrent map writes"), + ] if phrase in line), "panic-redacted") + facts["panicCategories"].append(category) + frame = re.match(r"^((?:k8s\.io/(?:kubernetes|apiserver|apiextensions-apiserver)|github\.com/google/cel-go)/[A-Za-z0-9_./@*()+-]+)\(", line) + if frame and frame[1] not in facts["publicFrames"]: + facts["publicFrames"].append(frame[1][:512]) + facts["panicCategories"] = sorted(set(facts["panicCategories"])) + facts["publicFrames"] = facts["publicFrames"][:40] + return facts + + +def controller_stack(context, name): + if name != "kube-controller-manager-kars-e2e-control-plane" or context != "kind-kars-e2e": + return {"available": False} + result = {"available": False, "current": {}, "previous": {}} + for previous in (False, True): + try: + command = ["kubectl", "--context", context, "--request-timeout=10s", "logs", + "-n", "kube-system", name, "--tail=240"] + if previous: + command.append("--previous") + output = subprocess.run(command, capture_output=True, text=True, timeout=15, check=False) + if output.returncode == 0: + result["available"] = True + result["previous" if previous else "current"] = public_stack_facts(output.stdout) + except (OSError, subprocess.TimeoutExpired): + pass + return result + + +def collect(port, policies, request): + report = {"scope": "controller-Pod-creation-only", "policies": [], "workloads": [], "events": []} + for name in sorted(policies): + code, obj = request(port, "GET", f"/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/{name}") + report["policies"].append(policy_status(obj, policies) if code == 200 else {"name": name, "httpStatus": code}) + owners = set() + for group, plural in [("apis/apps/v1", "deployments"), ("apis/apps/v1", "replicasets"), + ("api/v1", "pods")]: + code, body = request(port, "GET", f"/{group}/namespaces/kars-system/{plural}") + if code != 200 or not isinstance(body, dict): + report["workloads"].append({"kind": plural, "httpStatus": code}) + continue + for obj in body.get("items", [])[:256]: + metadata = obj.get("metadata", {}) + if not identifier(metadata.get("uid")): + continue + if metadata.get("name") == "kars-controller" or any( + owner.get("uid") in owners for owner in metadata.get("ownerReferences", [])): + owners.add(metadata.get("uid")) + obj = dict(obj, kind={"deployments": "Deployment", "replicasets": "ReplicaSet", "pods": "Pod"}[plural]) + report["workloads"].append(object_status(obj, policies)) + code, body = request(port, "GET", "/api/v1/namespaces/kars-system/events?fieldSelector=type%3DWarning") + if code == 200 and isinstance(body, dict): + for event in body.get("items", [])[:256]: + involved = event.get("involvedObject", {}) + if involved.get("uid") not in owners or event.get("reason") not in REASONS: + continue + entry = {"kind": involved.get("kind") if involved.get("kind") in ("Deployment", "ReplicaSet", "Pod") else None, + "name": identifier(involved.get("name")), + "reason": event.get("reason"), "count": event.get("count")} + entry.update(failure_facts(event.get("message"), policies)) + report["events"].append(entry) + code, body = request(port, "GET", "/api/v1/namespaces/kube-system/pods?labelSelector=component%3Dkube-controller-manager") + if code == 200 and isinstance(body, dict): + report["controlPlane"] = [control_plane_status(obj) for obj in body.get("items", [])[:4] + if obj.get("metadata", {}).get("name") == "kube-controller-manager-kars-e2e-control-plane"] + return report diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py new file mode 100644 index 000000000..6023668db --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -0,0 +1,288 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Actual disposable API Pod-creation proof without executing or pulling images.""" + +import argparse +import copy +import json +import os +from pathlib import Path +import re +import time + +from sre_authority.bootstrap_diagnostics import api_result, collect, controller_stack +from sre_authority.registration_schema import CONTEXT, command, kind_proxy, request, write_report as schema_report + +REPORT_PREFIX = "" + + +def write_report(root, filename, report): + schema_report(root, REPORT_PREFIX + filename, report) + + +def failure_site(error): + result = {"category": type(error).__name__} + frame = error.__traceback__ + while frame: + name = Path(frame.tb_frame.f_code.co_filename).name + if name in ("bootstrap_probe.py", "binding_probe.py", "bootstrap_cases.py", + "controller_update_probe.py", "collection_delete_probe.py"): + result.update(source=name, line=frame.tb_lineno) + frame = frame.tb_next + return result + + +PATHS = { + "CustomResourceDefinition": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions", + "ServiceAccount": "/api/v1/namespaces/{namespace}/serviceaccounts", + "ClusterRole": "/apis/rbac.authorization.k8s.io/v1/clusterroles", + "ClusterRoleBinding": "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings", + "Role": "/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles", + "RoleBinding": "/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings", + "ValidatingAdmissionPolicy": "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies", + "ValidatingAdmissionPolicyBinding": "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings", + "Deployment": "/apis/apps/v1/namespaces/{namespace}/deployments", +} + + +def builtin_documents(rendered): + documents = [] + for document in re.split(r"(?m)^---\s*\n", rendered): + kinds = re.findall(r"(?m)^kind:\s*([A-Za-z]+)\s*$", document) + if len(kinds) == 1 and kinds[0] in PATHS: + documents.append(document) + return "\n---\n".join(documents) + + +def converted_objects(raw): + # kubectl create -o json emits adjacent JSON objects for a multi-document + # input, rather than the List returned by kubectl get. + decoder, objects = json.JSONDecoder(), [] + while raw.strip(): + obj, end = decoder.raw_decode(raw.lstrip()) + raw = raw.lstrip()[end:] + objects.extend(obj.get("items", []) if obj.get("kind") == "List" else [obj]) + if not objects or any(obj.get("kind") not in PATHS for obj in objects): + raise RuntimeError("Public bootstrap chart contained unexpected converted objects") + return objects + + +def chart(root): + write_report(root, "bootstrap-stage.json", {"stage": "render-disabled-core"}) + rendered = command("bootstrap-render", [ + "helm", "template", "kars", str(root / "deploy/helm/kars"), "--namespace", "kars-system", + # Offline rendering cannot satisfy the deliberate live-source Helm lookup. + # The disabled-core stage emits the same controller and admission policies, + # without creating or executing an SRE source. + "--kube-version", "1.31.0", "--set", "sre.enabled=false", "--set", "sre.authorityStage=true", + "--set", "controller.replicas=1", "--set", "inferenceRouter.replicas=1", + "--set-string", "inferenceRouter.azure.openai.endpoint=https://e2e-fake.invalid/", + "--set-string", "foundry.endpoint=https://e2e-fake.invalid/", + "--set-string", "foundry.projectEndpoint=https://e2e-fake.invalid/", + ], root=root) + write_report(root, "bootstrap-stage.json", {"stage": "convert-public-builtins"}) + converted = command("bootstrap-conversion", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=builtin_documents(rendered)) + write_report(root, "bootstrap-stage.json", {"stage": "parse-public-objects"}) + return converted_objects(converted) + + +def safe_controller(obj): + if obj.get("kind") != "Deployment" or obj.get("metadata", {}).get("name") != "kars-controller": + raise AssertionError("Only the public controller template is a bootstrap fixture") + fixture = copy.deepcopy(obj) + fixture["metadata"]["namespace"] = "kars-system" + fixture["spec"]["replicas"] = 1 + pod = fixture["spec"]["template"]["spec"] + # Admission proof only: no node scheduling, image pulls or container execution. + pod["schedulerName"] = "kars-e2e-admission-never-schedule" + for container in pod.get("containers", []) + pod.get("initContainers", []): + container["image"] = "registry.invalid/kars-admission-proof:never" + container["imagePullPolicy"] = "Never" + return fixture + + +def preserved_json_candidate(objects): + candidate = copy.deepcopy(objects) + matches = [obj for obj in candidate if obj["kind"] == "CustomResourceDefinition" + and obj["metadata"]["name"] == "karssreactions.kars.azure.com"] + if len(matches) != 1: + raise RuntimeError("Expected one public SRE action schema") + schema = matches[0]["spec"]["versions"][0]["schema"]["openAPIV3Schema"] + params = schema["properties"]["spec"]["properties"]["action"]["properties"]["params"] + if params.get("type") != "object" or params.get("additionalProperties") is not True: + raise RuntimeError("API-evidenced boolean additionalProperties candidate precondition failed") + del params["additionalProperties"] + params["x-kubernetes-preserve-unknown-fields"] = True + return candidate + + +def upsert(port, obj, policies): + metadata = obj.get("metadata", {}) + path = PATHS[obj["kind"]].format(namespace=metadata.get("namespace", "kars-system")) + code, existing = request(port, "GET", f"{path}/{metadata['name']}") + method, accepted = ("PUT", 200) if code == 200 else ("POST", 201) + if code not in (200, 404): + return api_result(code, existing, policies) + desired = copy.deepcopy(obj) + if method == "PUT": + desired["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"] + path += "/" + metadata["name"] + code, body = request(port, method, path, desired) + result = api_result(code, body, policies) + result["accepted"] = code == accepted + result["kind"], result["name"] = obj["kind"], metadata["name"] + return result + + +def exercise(root, port, objects, policies, wait_seconds=90, retirement=False): + results = [] + for name in ("kars-system", "kars-sre"): + code, body = request(port, "POST", "/api/v1/namespaces", { + "apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}}) + if code not in (201, 409): + raise RuntimeError("Disposable bootstrap namespace unavailable") + retirement_state = None + if retirement: + from sre_authority.binding_probe import seed + retirement_state = seed(port, root) + for kind in PATHS: + if kind == "Deployment": + continue + for obj in objects: + if obj["kind"] != kind: + continue + result = upsert(port, obj, policies) + results.append(result) + if not result.get("accepted"): + write_report(root, "bootstrap-install.json", {"operations": results}) + raise RuntimeError("Disposable public bootstrap admission install failed") + deadline = time.monotonic() + wait_seconds + while time.monotonic() < deadline: + snapshot = collect(port, policies, request) + if all(entry.get("typeChecked") and entry.get("generation") == entry.get("observedGeneration") + for entry in snapshot["policies"]): + break + time.sleep(1) + else: + write_report(root, "bootstrap-before-create.json", snapshot) + # Still collect actual Pod/ReplicaSet admission evidence after the + # bounded observation wait. An unrelated policy must not hide the + # creation failure; the unobserved-policy gate remains fatal below. + observed = all(entry.get("typeChecked") and entry.get("generation") == entry.get("observedGeneration") + for entry in snapshot["policies"]) + write_report(root, "bootstrap-before-create.json", snapshot) + templates = [obj for obj in objects if obj["kind"] == "Deployment" + and obj["metadata"]["name"] == "kars-controller"] + if len(templates) != 1: + raise RuntimeError("Expected one public controller Deployment") + fixture = safe_controller(templates[0]) + template = fixture["spec"]["template"] + pod = {"apiVersion": "v1", "kind": "Pod", + "metadata": {"name": "kars-controller-admission-direct", "namespace": "kars-system", + "labels": template["metadata"]["labels"]}, "spec": template["spec"]} + code, body = request(port, "POST", "/api/v1/namespaces/kars-system/pods?dryRun=All", pod) + direct = api_result(code, body, policies) + result = upsert(port, fixture, policies) + write_report(root, "bootstrap-create.json", {"directPodDryRun": direct, "deployment": result}) + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + snapshot = collect(port, policies, request) + if any(obj.get("kind") == "Pod" for obj in snapshot["workloads"]): + break + time.sleep(1) + write_report(root, "bootstrap-after-create.json", snapshot) + if code != 201 or not result.get("accepted") or not any(obj.get("kind") == "Pod" for obj in snapshot["workloads"]): + raise RuntimeError("Real controller Deployment/ReplicaSet did not create an admission-only Pod") + write_report(root, "bootstrap-result.json", {"podCreation": "accepted", + "workloadExecution": "not-attempted", "readiness": "not-claimed", + "allPoliciesObservedBeforeCreate": observed}) + if not observed: + raise RuntimeError("Public admission policy observation timed out; Pod evidence was still collected") + return retirement_state + + +def main(root, diagnostics_only, candidate=False, retirement=False): + global REPORT_PREFIX + REPORT_PREFIX = "candidate-" if candidate else "" + with kind_proxy(root) as (port, version): + write_report(root, "bootstrap-versions.json", {"apiServer": version, "context": CONTEXT}) + objects = chart(root) + if candidate: + objects = preserved_json_candidate(objects) + policies = {obj["metadata"]["name"]: obj for obj in objects + if obj["kind"] == "ValidatingAdmissionPolicy"} + if not policies: + raise RuntimeError("Public admission policies were not rendered") + if diagnostics_only: + write_report(root, "bootstrap-install-failure.json", collect(port, policies, request)) + write_report(root, "bootstrap-controller-stack.json", controller_stack( + CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) + else: + try: + media = [] + path = ("/api/v1/namespaces/kube-system/pods/kube-controller-manager-kars-e2e-control-plane/log" + "?container=kube-controller-manager&tailLines=1&limitBytes=4096") + for accept in ("text/plain", "application/json", "*/*"): + code, _ = request(port, "GET", path, accept=accept) + media.append({"accept": accept, "httpStatus": code}) + write_report(root, "bootstrap-log-media.json", {"cases": media}) + if next(case["httpStatus"] for case in media if case["accept"] == "application/json") != 200: + raise RuntimeError("Actual API log media precondition failed") + state = exercise(root, port, objects, policies, wait_seconds=180 if candidate else 90, + retirement=retirement and not candidate) + from sre_authority.bootstrap_cases import admission_cases + cases = admission_cases(port, policies) + write_report(root, "bootstrap-admission-cases.json", {"cases": cases}) + if not all(case["matched"] for case in cases): + raise RuntimeError("Schema failed intended ordinary/private admission outcomes") + if state: + from sre_authority.binding_probe import prove + prove(root, port, state, objects, + lambda facts: write_report(root, "bootstrap-binding-retirement.json", facts)) + from sre_authority.bootstrap_cases import deployment_controller_cases, namespace_cleanup_cases, private_controller_chain + controller_cases = deployment_controller_cases(port, policies) + write_report(root, "bootstrap-workload-controller.json", {"cases": controller_cases}) + if not all(case["matched"] for case in controller_cases): + raise RuntimeError("Built-in Deployment controller cannot create the private SRE ReplicaSet") + parent = private_controller_chain(port, policies, + lambda facts: write_report(root, "bootstrap-private-controller-chain.json", facts)) + from sre_authority.controller_update_probe import cases as update_cases + updates = update_cases(port, policies, parent) + write_report(root, "bootstrap-controller-update-ownerrefs.json", {"cases": updates}) + if not all(case["matched"] for case in updates): + raise RuntimeError("Private workload UPDATE/owner-reference boundary failed") + cleanup_cases = namespace_cleanup_cases(port, policies, state["consumer"] if state else None) + write_report(root, "bootstrap-namespace-cleanup.json", {"cases": cleanup_cases}) + if not all(case["matched"] for case in cleanup_cases): + raise RuntimeError("Canonical consumer cleanup authority differs from the expected boundary") + from sre_authority.collection_delete_probe import cases as collection_cases + collections = collection_cases(port, policies) + write_report(root, "bootstrap-collection-delete.json", {"cases": collections}) + if not all(case["matched"] for case in collections): + raise RuntimeError("Collection DELETE must preserve ordinary cleanup and protected-object denial") + finally: + write_report(root, "bootstrap-final.json", collect(port, policies, request)) + write_report(root, "bootstrap-controller-stack.json", controller_stack( + CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) + + +if __name__ == "__main__": + os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--diagnostics-only", action="store_true") + parser.add_argument("--json-params-candidate", action="store_true", + help="Diagnose only the API-evidenced nil-schema adapter candidate; never edit production") + parser.add_argument("--retirement-bind-proof", action="store_true", + help="Prove controller-principal retirement dry-runs with a test-only exact reader bind grant") + args = parser.parse_args() + try: + main(Path(__file__).resolve().parents[3], args.diagnostics_only, args.json_params_candidate, + args.retirement_bind_proof) + except Exception as error: + print(f"SRE-BOOTSTRAP-FAIL category={type(error).__name__}", flush=True) + write_report(Path(__file__).resolve().parents[3], "bootstrap-failure-site.json", failure_site(error)) + raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py new file mode 100644 index 000000000..cd18bb1dc --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -0,0 +1,445 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +from pathlib import Path +import unittest +from unittest.mock import patch + +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, exception_summary, failure_facts, firewall_summary, object_status, policy_status, probe_command_result, public_stack_facts, router_log_summary, router_readiness_facts +from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller + +POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ + {"message": "Private SRE material requires authority"}]}}} + + +class BootstrapProofTests(unittest.TestCase): + def test_policy_compile_diagnostics_keep_only_known_fields_and_token_categories(self): + body = {"kind": "Status", "reason": "Invalid", "details": { + "group": "admissionregistration.k8s.io", "kind": "ValidatingAdmissionPolicy", + "name": "kars-sre-private-mounts", "causes": [ + {"field": "spec.matchConditions[0].expression", + "message": "compilation failed: undefined field 'metadata'; optional overload do-not-publish"}, + {"field": "spec.arbitrary", "message": "do-not-publish"}, + ]}} + facts = api_result(422, body, POLICIES) + self.assertEqual(facts["compilationCauses"][0]["knownTokens"], ["metadata"]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + body["details"]["name"] = "unrelated" + self.assertNotIn("compilationCauses", api_result(422, body, POLICIES)) + + def test_namespace_cleanup_proof_retains_guard_and_fences_all_deletes(self): + from sre_authority.bootstrap_cases import namespace_cleanup_cases + deployment = {"metadata": {"uid": "owned", "resourceVersion": "2"}} + responses = [(404, {}), (201, deployment), (200, {"metadata": {"uid": "namespace-controller"}}), + (200, deployment), (200, {"metadata": {"uid": "kars-controller"}}), + (200, deployment), (200, deployment), (200, {})] + with patch("sre_authority.bootstrap_cases.request", side_effect=responses) as api, \ + patch("sre_authority.bootstrap_cases.as_tenant", side_effect=[ + (403, {"kind": "Status", "reason": "Forbidden", + "message": "kars-sre-consumer-authority denied do-not-publish"}), + (200, {"kind": "Status", "status": "Success"})]) as actor: + cases = namespace_cleanup_cases(1, {"kars-sre-consumer-authority": {}}) + self.assertTrue(all(case["matched"] for case in cases)) + self.assertNotIn("do-not-publish", json.dumps(cases)) + for call in actor.call_args_list: + self.assertEqual(call.kwargs["method"], "DELETE") + self.assertNotIn("?", call.args[1]) + self.assertEqual(call.args[2]["dryRun"], ["All"]) + self.assertEqual(call.args[2]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) + self.assertEqual(api.call_args_list[-1].args[1], "DELETE") + self.assertEqual(api.call_args_list[-1].args[3]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) + + def test_cleanup_proof_never_adopts_an_unexpected_existing_consumer(self): + from sre_authority.bootstrap_cases import namespace_cleanup_cases + owned = {"metadata": {"uid": "owned"}, "spec": {"replicas": 0}} + for current, expected in ((owned, None), + ({"metadata": {"uid": "other"}, "spec": owned["spec"]}, owned), + ({"metadata": owned["metadata"], "spec": {"replicas": 1}}, owned)): + with patch("sre_authority.bootstrap_cases.request", return_value=(200, current)) as api, \ + self.assertRaises(RuntimeError): + namespace_cleanup_cases(1, {}, expected) + api.assert_called_once() + + def test_http_failure_summary_reports_status_and_checked_source_not_body_url_or_headers(self): + root = Path(__file__).resolve().parents[3] + class Response: + status_code = 401 + def json(self): + return {"kind": "Status", "reason": "SREProxyDenied", + "message": "An SRE proxy credential is required", "data": "do-not-publish"} + error = RuntimeError("do-not-publish URL/header/credential") + error.response = Response() + facts = exception_summary(error, root) + self.assertEqual(facts["httpStatus"], 401) + self.assertTrue(facts["responseSite"]["source"].endswith("sre_proxy/mod.rs")) + self.assertNotIn("do-not-publish", json.dumps(facts)) + error.response.json = lambda: {"kind": "Status", "message": "do-not-publish"} + self.assertNotIn("responseSite", exception_summary(error, root)) + self.assertNotIn("do-not-publish", json.dumps(exception_summary(error, root))) + + def test_firewall_summary_does_not_publish_rules_comments_addresses_or_unknown_chains(self): + text = """*filter +:OUTPUT ACCEPT [12:640] +[4:240] -A OUTPUT -m owner --uid-owner 1000 -j DROP +[3:120] -A OUTPUT -m owner ! --uid-owner 1001 -o lo -j ACCEPT +[2:80] -A do-not-publish -s do-not-publish -m comment --comment do-not-publish -j DROP +COMMIT +""" + value = firewall_summary(text) + self.assertEqual(value["policies"], [{"table": "filter", "chain": "OUTPUT", "policy": "ACCEPT", "packets": 12}]) + self.assertEqual(value["rules"][0]["owner"], 1000) + self.assertTrue(value["rules"][1]["ownerNegated"]) + self.assertEqual(value["rules"][2]["chain"], "custom") + self.assertNotIn("do-not-publish", json.dumps(value)) + + def test_router_startup_summary_reports_only_verified_source_coordinates(self): + root = Path(__file__).resolve().parents[3] + text = "\n".join([ + json.dumps({"fields": {"message": "kars Inference Router starting", "token": "do-not-publish"}}), + json.dumps({"fields": {"message": "do-not-publish"}}), + "do-not-publish plaintext", + ]) + result = router_log_summary(text, root) + self.assertEqual(result["lines"], 3) + self.assertEqual(result["jsonEvents"], 2) + self.assertEqual(len(result["sourceSites"]), 1) + self.assertEqual(result["sourceSites"][0]["source"], "inference-router/src/main.rs") + self.assertNotIn("do-not-publish", json.dumps(result)) + + def test_actual_json_tracing_format_is_parsed_without_publishing_other_fields(self): + events = [ + {"message": "SRE authority transport failure", "stage": "registration", + "timed_out": True, "connect_error": False, "token": "do-not-publish"}, + {"message": "SRE authority request denied", "stage": "privacy-review", "http_status": 403}, + {"message": "SRE readiness authority slow", "authorized": True, "elapsed_seconds": 8}, + {"message": "SRE transport progress", "stage": "tls-accepted", "peer": "do-not-publish"}, + {"message": "SRE authority progress", "stage": "registration", + "step": "request", "url": "do-not-publish"}, + {"message": "unrelated do-not-publish", "stage": "namespace"}, + {"message": "SRE authority request denied", "stage": "do-not-publish", "http_status": True}, + ] + text = "\n".join(json.dumps({"fields": fields, "span": {"token": "do-not-publish"}}) for fields in events) + facts = router_readiness_facts(text) + self.assertEqual(facts, [ + {"stage": "registration", "timed_out": True, "connect_error": False}, + {"stage": "privacy-review", "httpStatus": 403}, + {"authorized": True, "elapsedSeconds": 8}, + {"stage": "tls-accepted"}, + {"stage": "registration", "step": "request"}, + ]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + self.assertEqual(router_readiness_facts(json.dumps(events[0])), [facts[0]]) + + def test_router_readiness_logs_expose_only_fixed_stage_status_and_timeout_facts(self): + text = ( + '\x1b[33mSRE authority transport failure\x1b[0m stage="registration" timed_out=true connect_error=false token=do-not-publish\n' + 'SRE authority request denied stage="privacy-review" http_status=403 body=do-not-publish\n' + 'SRE readiness authority rejected category="authority-transport" do-not-publish\n' + 'SRE readiness authority slow elapsed_seconds=20 authorized=false do-not-publish\n' + 'unrelated log stage="source" do-not-publish\n' + 'SRE authority request denied stage="do-not-publish" http_status=999\n' + ) + facts = router_readiness_facts(text) + self.assertEqual(facts, [ + {"stage": "registration", "timed_out": True, "connect_error": False}, + {"stage": "privacy-review", "httpStatus": 403}, + {"category": "authority-transport"}, + {"authorized": False, "elapsedSeconds": 20}, + ]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + + def test_probe_diagnostics_never_publish_executable_output(self): + for code, output, category in ( + (127, "exec: executable file not found in $PATH do-not-publish", "executable-not-found"), + (1, "", "probe-not-ready"), + (1, "command terminated with exit code 1\n", "probe-not-ready"), + (0, "do-not-publish", "succeeded"), + (1, "do-not-publish", "unclassified"), + ): + facts = probe_command_result(code, output) + self.assertEqual(facts, {"exitCode": code, "category": category}) + self.assertNotIn("do-not-publish", json.dumps(facts)) + + def test_failure_metadata_keeps_public_cause_not_body_or_credentials(self): + message = ('Error creating Pod: kars-sre-private-mounts evaluation failed: no such key: namespace; ' + 'token=do-not-publish argv=do-not-publish') + result = object_status({ + "kind": "ReplicaSet", "metadata": {"name": "kars-controller-abc", "uid": "rs-uid", + "annotations": {"credential": "do-not-publish"}}, + "spec": {"template": {"spec": {"containers": [{"env": ["do-not-publish"]}]}}}, + "status": {"replicas": 0, "conditions": [{"type": "ReplicaFailure", "status": "True", + "reason": "FailedCreate", "message": message}]}, + }, POLICIES) + self.assertNotIn("do-not-publish", json.dumps(result)) + self.assertEqual(result["conditions"][0]["reason"], "FailedCreate") + self.assertEqual(result["conditions"][0]["policies"], ["kars-sre-private-mounts"]) + self.assertEqual(result["conditions"][0]["missingFields"], ["namespace"]) + self.assertNotIn("do-not-publish", json.dumps(api_result( + 403, {"kind": "Status", "reason": "Forbidden", "message": message, "details": "do-not-publish"}, POLICIES))) + + def test_type_warning_scope_is_known_public_policy_field_only(self): + obj = {"metadata": {"name": "kars-sre-private-mounts", "generation": 3}, + "status": {"observedGeneration": 3, "typeChecking": {"expressionWarnings": [ + {"fieldRef": "spec.variables[0].expression", "warning": "undefined field namespace"}, + {"fieldRef": "spec.containers[0].env", "warning": "do-not-publish"}, + ]}}} + result = policy_status(obj, POLICIES) + self.assertTrue(result["typeChecked"]) + self.assertEqual(len(result["warnings"]), 1) + self.assertNotIn("do-not-publish", json.dumps(result)) + obj["metadata"]["name"] = "unrelated-policy" + with self.assertRaises(AssertionError): + policy_status(obj, POLICIES) + + def test_control_plane_metadata_and_go_frames_never_publish_logs_or_arguments(self): + pod = {"metadata": {"name": "kube-controller-manager-kars-e2e-control-plane"}, + "spec": {"containers": [{"args": ["do-not-publish"]}]}, + "status": {"containerStatuses": [{"name": "kube-controller-manager", "ready": False, + "restartCount": 4, "state": {"waiting": {"message": "do-not-publish"}}, + "lastState": {"terminated": {"exitCode": 2, "reason": "Error", "message": "do-not-publish"}}}]}} + status = control_plane_status(pod) + self.assertEqual(status["containers"][0]["restartCount"], 4) + self.assertNotIn("do-not-publish", json.dumps(status)) + facts = public_stack_facts( + "panic: runtime error: invalid memory address or nil pointer dereference\n" + "token=do-not-publish request body do-not-publish\n" + "k8s.io/apiserver/pkg/admission/plugin/policy/validating.(*TypeChecker).Check(do-not-publish)\n" + "panic: do-not-publish\n") + self.assertIn("nil-pointer", facts["panicCategories"]) + self.assertIn("panic-redacted", facts["panicCategories"]) + self.assertEqual(facts["publicFrames"], [ + "k8s.io/apiserver/pkg/admission/plugin/policy/validating.(*TypeChecker).Check"]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + + def test_no_scheduling_pulls_or_execution_but_original_controller_shape_remains(self): + original = {"kind": "Deployment", "metadata": {"name": "kars-controller"}, + "spec": {"replicas": 0, "template": {"spec": { + "serviceAccountName": "kars-controller", "automountServiceAccountToken": True, + "containers": [{"name": "controller", "image": "original", "env": [{"name": "X", "value": "Y"}]}], + "initContainers": [{"name": "init", "image": "original-init"}], + }}}} + before = copy.deepcopy(original) + result = safe_controller(original) + self.assertEqual(original, before) + pod = result["spec"]["template"]["spec"] + self.assertEqual(result["spec"]["replicas"], 1) + self.assertEqual(pod["serviceAccountName"], "kars-controller") + self.assertEqual(pod["schedulerName"], "kars-e2e-admission-never-schedule") + self.assertEqual(pod["containers"][0]["env"], [{"name": "X", "value": "Y"}]) + self.assertTrue(all(c["imagePullPolicy"] == "Never" and c["image"].startswith("registry.invalid/") + for c in pod["containers"] + pod["initContainers"])) + with self.assertRaises(AssertionError): + safe_controller({"kind": "Deployment", "metadata": {"name": "unrelated"}}) + + def test_chart_selection_never_executes_other_workloads_or_loads_secret_data(self): + rendered = "---\nkind: Secret\nmetadata:\n name: private\nstringData:\n token: do-not-publish\n" + rendered += "---\nkind: Job\nmetadata:\n name: execute\n---\nkind: ValidatingAdmissionPolicy\nmetadata:\n name: public\n" + result = builtin_documents(rendered) + self.assertNotIn("do-not-publish", result) + self.assertNotIn("kind: Job", result) + self.assertIn("kind: ValidatingAdmissionPolicy", result) + + def test_kubectl_multiple_json_objects_and_list_are_both_parsed(self): + first = {"kind": "ServiceAccount", "metadata": {"name": "kars-controller"}} + second = {"kind": "ValidatingAdmissionPolicy", "metadata": {"name": "public"}} + self.assertEqual(converted_objects(json.dumps(first) + "\n" + json.dumps(second)), [first, second]) + self.assertEqual(converted_objects(json.dumps({"kind": "List", "items": [first, second]})), [first, second]) + with self.assertRaises(RuntimeError): + converted_objects(json.dumps({"kind": "Secret", "data": "do-not-publish"})) + + def test_candidate_changes_only_known_public_json_params_representation(self): + params = {"type": "object", "additionalProperties": True, "description": "public"} + action = {"kind": "CustomResourceDefinition", "metadata": {"name": "karssreactions.kars.azure.com"}, + "spec": {"versions": [{"schema": {"openAPIV3Schema": {"properties": {"spec": {"properties": { + "action": {"properties": {"params": params}}, "approval": {"type": "object"} + }}}}}}]}} + policy = {"kind": "ValidatingAdmissionPolicy", "metadata": {"name": "kars-sre-pending-proposals"}, + "spec": {"validations": [{"expression": "object.spec.approval.state == 'Pending'"}]}} + source = [action, policy] + before = copy.deepcopy(source) + changed = preserved_json_candidate(source) + self.assertEqual(source, before) + self.assertEqual(changed[1], policy) + actual = changed[0]["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]["action"]["properties"]["params"] + self.assertEqual(actual, {"type": "object", "x-kubernetes-preserve-unknown-fields": True, "description": "public"}) + params["additionalProperties"] = {"type": "string"} + with self.assertRaises(RuntimeError): + preserved_json_candidate(source) + + def test_arbitrary_failure_is_not_security_proof_in_candidate_cases(self): + from sre_authority.bootstrap_cases import admission_cases + with patch("sre_authority.bootstrap_cases.request", return_value=(201, {})), \ + patch("sre_authority.bootstrap_cases.upsert", return_value={"accepted": True}), \ + patch("sre_authority.bootstrap_cases.as_tenant", return_value=( + 403, {"kind": "Status", "reason": "Forbidden", "message": "unrelated do-not-publish"})): + cases = admission_cases(1, POLICIES) + self.assertTrue(cases) + self.assertFalse(any(case["matched"] for case in cases)) + self.assertNotIn("do-not-publish", json.dumps(cases)) + + def test_builtin_controller_proof_preserves_real_identity_and_cluster_scope(self): + from sre_authority.bootstrap_cases import DEPLOYMENT_CONTROLLER, deployment_controller_cases + responses = [(200, {"metadata": {"uid": "actual-api-uid"}}), + (201, {"status": {"allowed": True}}), + (201, {"status": {"allowed": False}}), + (201, {"status": {"allowed": False}})] + with patch("sre_authority.bootstrap_cases.request", side_effect=responses) as api, \ + patch("sre_authority.bootstrap_cases.as_tenant", return_value=(201, {"kind": "ReplicaSet"})) as actor: + cases = deployment_controller_cases(1, POLICIES) + self.assertTrue(all(case["matched"] for case in cases)) + self.assertEqual(cases[0]["authorization"], { + "createReplicaSetsClusterWide": True, "createPodsClusterWide": False, "useRegistrar": False}) + for call in api.call_args_list[1:]: + spec = call.args[3]["spec"] + self.assertEqual(spec["user"], DEPLOYMENT_CONTROLLER) + self.assertNotIn("namespace", spec["resourceAttributes"]) + for call in actor.call_args_list: + self.assertEqual(call.kwargs["user"], DEPLOYMENT_CONTROLLER) + self.assertTrue(call.args[1].endswith("?dryRun=All")) + pod = call.args[2]["spec"]["template"]["spec"] + self.assertEqual(pod["schedulerName"], "kars-e2e-admission-never-schedule") + self.assertEqual(pod["containers"][0]["imagePullPolicy"], "Never") + + def test_private_controller_replicaset_denial_is_a_failure_not_a_security_pass(self): + from sre_authority.bootstrap_cases import deployment_controller_cases + policies = {"kars-sre-private-workloads": {}} + for code in (403, 404, 422, 500): + responses = [(200, {"metadata": {"uid": "actual-api-uid"}})] + [ + (201, {"status": {"allowed": allowed}}) for allowed in (True, False, False)] + with patch("sre_authority.bootstrap_cases.request", side_effect=responses), \ + patch("sre_authority.bootstrap_cases.as_tenant", side_effect=[ + (201, {"kind": "ReplicaSet"}), + (code, {"kind": "Status", "reason": "Forbidden", + "message": "kars-sre-private-workloads forbids do-not-publish"})]): + cases = deployment_controller_cases(1, policies) + self.assertTrue(cases[0]["matched"]) + self.assertFalse(cases[1]["matched"]) + self.assertEqual(cases[1]["expectedStatus"], 201) + self.assertNotIn("do-not-publish", json.dumps(cases)) + + def test_controller_proof_requires_actual_account_and_valid_authorization_response(self): + from sre_authority.bootstrap_cases import deployment_controller_cases + for responses in ([(404, {})], [(200, {"metadata": {}})], + [(200, {"metadata": {"uid": "uid"}}), (403, {})], + [(200, {"metadata": {"uid": "uid"}}), (201, {"status": {"allowed": "true"}})]): + with patch("sre_authority.bootstrap_cases.request", side_effect=responses), \ + patch("sre_authority.bootstrap_cases.as_tenant") as actor, self.assertRaises(RuntimeError): + deployment_controller_cases(1, POLICIES) + actor.assert_not_called() + + def private_chain_api(self, *, pod_change=None, replaced=False, no_child=False): + created = {} + def api(_port, method, path, obj=None): + if method == "POST": + created.update(copy.deepcopy(obj)) + created["metadata"]["uid"] = "deployment-uid" + return 201, created + if "/deployments/" in path: + current = copy.deepcopy(created) + if replaced: + current["metadata"]["uid"] = "replacement" + current["status"] = {"conditions": [{"type": "Progressing", "status": "False", + "reason": "ReplicaSetCreateError", "message": "kars-sre-private-workloads forbidden do-not-publish"}]} + return 200, current + if "/replicasets?" in path: + return 200, {"items": [{"metadata": {"name": "owned-rs", "uid": "replicaset-uid", + "ownerReferences": [{"uid": "deployment-uid", "controller": True}]}}]} + if "/pods?" in path: + pod = {"metadata": {"name": "owned-pod", "uid": "pod-uid", + "ownerReferences": [{"uid": "replicaset-uid", "controller": True}]}, + "spec": copy.deepcopy(created["spec"]["template"]["spec"])} + if pod_change: + pod_change(pod) + return 200, {"items": [] if no_child else [pod, {"metadata": {"name": "do-not-publish", + "uid": "foreign", "ownerReferences": [{"uid": "not-ours", "controller": True}]}}]} + raise AssertionError("Unexpected private chain API request") + return api + + def test_actual_private_chain_requires_deployment_replicaset_pod_uid_ownership(self): + from sre_authority.bootstrap_cases import private_controller_chain + reports = [] + with patch("sre_authority.bootstrap_cases.request", side_effect=self.private_chain_api()) as api: + private_controller_chain(1, {"kars-sre-private-workloads": {}}, reports.append) + self.assertTrue(reports[0]["privateMountPreserved"]) + self.assertTrue(reports[0]["noWorkloadExecution"]) + self.assertEqual([pod["uid"] for pod in reports[0]["pods"]], ["pod-uid"]) + self.assertNotIn("do-not-publish", json.dumps(reports)) + self.assertEqual([call.args[1] for call in api.call_args_list], ["POST", "GET", "GET", "GET"]) + self.assertTrue(api.call_args_list[0].args[2].endswith("/deployments")) + + def test_private_chain_rejects_replacement_missing_mount_or_execution(self): + from sre_authority.bootstrap_cases import private_controller_chain + variants = [ + {"replaced": True}, + {"pod_change": lambda pod: pod["spec"].update(nodeName="scheduled-node")}, + {"pod_change": lambda pod: pod["spec"].pop("volumes")}, + {"pod_change": lambda pod: pod["spec"]["containers"][0].pop("volumeMounts")}, + ] + for variant in variants: + with patch("sre_authority.bootstrap_cases.request", side_effect=self.private_chain_api(**variant)), \ + self.subTest(variant=variant), self.assertRaises(RuntimeError): + private_controller_chain(1, POLICIES, lambda _report: None) + + def test_private_chain_timeout_reports_sanitized_blocker_not_success(self): + from sre_authority.bootstrap_cases import private_controller_chain + for variant in ({"no_child": True}, + {"pod_change": lambda pod: pod["metadata"]["ownerReferences"][0].update(uid="foreign")}, + {"pod_change": lambda pod: pod["metadata"]["ownerReferences"][0].update(controller=False)}): + reports = [] + with patch("sre_authority.bootstrap_cases.request", side_effect=self.private_chain_api(**variant)), \ + patch("sre_authority.bootstrap_cases.time.monotonic", side_effect=[0, 1, 46]), \ + patch("sre_authority.bootstrap_cases.time.sleep"), self.assertRaises(RuntimeError): + private_controller_chain(1, {"kars-sre-private-workloads": {}}, reports.append) + self.assertFalse(reports[0]["privateMountPreserved"]) + self.assertFalse(reports[0]["noWorkloadExecution"]) + self.assertNotIn("do-not-publish", json.dumps(reports)) + + def test_collection_tracks_real_uid_chain_without_logging_other_pods(self): + def request(_port, _method, path): + if path.endswith("/deployments"): + return 200, {"items": [{"metadata": {"name": "kars-controller", "uid": "dep"}}]} + if path.endswith("/replicasets"): + return 200, {"items": [{"metadata": {"name": "kars-controller-rs", "uid": "rs", + "ownerReferences": [{"uid": "dep"}]}, "status": {"replicas": 0}}]} + if path.endswith("/pods"): + return 200, {"items": [{"metadata": {"name": "created", "uid": "pod", + "ownerReferences": [{"uid": "rs"}]}}, + {"metadata": {"name": "unrelated", "uid": "other"}, "spec": {"private": "do-not-publish"}}]} + return 200, {"items": []} + result = collect(1, {}, request) + self.assertEqual([o["kind"] for o in result["workloads"]], ["Deployment", "ReplicaSet", "Pod"]) + self.assertNotIn("do-not-publish", json.dumps(result)) + self.assertNotIn("unrelated", json.dumps(result)) + + def test_observation_timeout_collects_pod_evidence_but_still_fails(self): + deployment = {"kind": "Deployment", "metadata": {"name": "kars-controller"}, + "spec": {"template": {"metadata": {"labels": {}}, "spec": {"containers": []}}}} + unobserved = {"policies": [{"generation": 1, "typeChecked": False}], "workloads": []} + created = dict(unobserved, workloads=[{"kind": "Pod"}]) + with patch("sre_authority.bootstrap_probe.request", return_value=(201, {})) as api, \ + patch("sre_authority.bootstrap_probe.upsert", return_value={"accepted": True}), \ + patch("sre_authority.bootstrap_probe.collect", side_effect=[unobserved, created]), \ + patch("sre_authority.bootstrap_probe.time.monotonic", side_effect=[0, 1, 91, 100, 101]), \ + patch("sre_authority.bootstrap_probe.time.sleep"), \ + patch("sre_authority.bootstrap_probe.write_report") as report: + with self.assertRaisesRegex(RuntimeError, "observation timed out"): + exercise(Path("."), 1, [deployment], POLICIES) + self.assertTrue(any(call.args[2].endswith("pods?dryRun=All") for call in api.call_args_list)) + result = next(call.args[2] for call in report.call_args_list if call.args[1] == "bootstrap-result.json") + self.assertEqual(result["podCreation"], "accepted") + self.assertFalse(result["allPoliciesObservedBeforeCreate"]) + self.assertEqual(result["readiness"], "not-claimed") + + def test_failure_diagnostics_precede_teardown_without_pod_spec_or_log_dump(self): + source = (Path(__file__).resolve().parents[1] / "run.sh").read_text() + install = source.split("install_crds() {", 1)[1].split("\nteardown()", 1)[0] + self.assertIn("sre_authority.bootstrap_probe --diagnostics-only", install) + self.assertNotIn("kubectl describe pod", install) + self.assertNotIn("kubectl logs", install) + self.assertIn("return 1", install) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/collection_delete_probe.py b/tests/e2e/sre_authority/collection_delete_probe.py new file mode 100644 index 000000000..c8875709b --- /dev/null +++ b/tests/e2e/sre_authority/collection_delete_probe.py @@ -0,0 +1,115 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Real nameless collection DELETE checks without removing any guarded fixture.""" + +import copy +import time +from urllib.parse import quote + +from .bootstrap_cases import as_tenant +from .bootstrap_diagnostics import api_result +from .common import require +from .registration_schema import request + +NAMESPACE_CONTROLLER = "system:serviceaccount:kube-system:namespace-controller" + + +def stable_object(value): + value = copy.deepcopy(value) + value.pop("status", None) + value.get("metadata", {}).pop("resourceVersion", None) + value.get("metadata", {}).pop("managedFields", None) + return value + + +def dry_run_delete(port, collection, item_path, current, admin): + for attempt in range(3): + options = {"apiVersion": "v1", "kind": "DeleteOptions", "dryRun": ["All"], + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}} + if admin: + code, body = request(port, "DELETE", collection, options) + else: + code, body = as_tenant(port, collection, options, + user=NAMESPACE_CONTROLLER, method="DELETE") + if (code != 409 or not isinstance(body, dict) or body.get("reason") != "Conflict" + or attempt == 2): + return code, body + status, refreshed = request(port, "GET", item_path) + require(status == 200 and stable_object(refreshed) == stable_object(current), + "Collection dry-run target changed beyond controller status; no retry permitted") + if refreshed["metadata"]["resourceVersion"] == current["metadata"]["resourceVersion"]: + return code, body + current = refreshed + time.sleep(0.2) + raise AssertionError("Collection dry-run exceeded its fixed attempt bound") + + +def cases(port, policies): + fixtures, reports = [], [] + primary_failure = False + definitions = [ + ("v1", "ServiceAccount", "/api/v1/namespaces/kars-sre/serviceaccounts", + "sre-api-router", "kars-sre-private-identity", {"automountServiceAccountToken": False}), + ("rbac.authorization.k8s.io/v1", "Role", "/apis/rbac.authorization.k8s.io/v1/namespaces/kars-sre/roles", + "sre-api-self-renew", "kars-sre-role-authority", {"rules": []}), + ("apps/v1", "Deployment", "/apis/apps/v1/namespaces/kars-sre/deployments", + "sre", "kars-sre-consumer-authority", {"spec": { + "replicas": 0, "paused": True, "selector": {"matchLabels": {"app": "e2e-collection"}}, + "template": {"metadata": {"labels": {"app": "e2e-collection"}}, "spec": { + "automountServiceAccountToken": False, "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]}}}}), + ] + try: + for version, kind, path, protected_name, policy, fields in definitions: + for protected in (False, True): + name = protected_name if protected else "e2e-collection-" + kind.lower() + obj = {"apiVersion": version, "kind": kind, + "metadata": {"name": name, "namespace": "kars-sre"}, **fields} + code, created = request(port, "POST", path, obj) + require(code == 201 and created.get("metadata", {}).get("uid"), + "Collection proof fixture CREATE failed; no adoption allowed") + item_path = path + "/" + name + fixtures.append((item_path, created)) + collection = path + "?fieldSelector=" + quote("metadata.name=" + name, safe="") + for admin in (False, True) if protected else (False,): + code, current = request(port, "GET", item_path) + require(code == 200 and current["metadata"]["uid"] == created["metadata"]["uid"], + "Collection proof fixture identity changed") + code, body = dry_run_delete(port, collection, item_path, current, admin) + expected = 403 if protected and not admin else 200 + result = api_result(code, body, policies) + result.update({ + "case": f"{kind}-{'protected' if protected else 'ordinary'}-{'admin' if admin else 'namespace-controller'}-collection", + "expectedStatus": expected, + "matched": code == expected and (expected != 403 or ( + policy in result.get("policies", []) + and not set(result.get("categories", [])) & {"evaluation", "no-such-key", "compilation"})), + }) + reports.append(result) + code, after = request(port, "GET", item_path) + require(code == 200 and after["metadata"]["uid"] == created["metadata"]["uid"] + and not after["metadata"].get("deletionTimestamp"), + "Collection DELETE dry-run mutated a fixture") + return reports + except BaseException: + primary_failure = True + raise + finally: + for path, original in reversed(fixtures): + try: + code, current = request(port, "GET", path) + require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"], + "Collection fixture cleanup identity changed") + require(stable_object(current) == stable_object(original), + "Collection fixture content changed; cleanup preserved it") + code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}}) + require(code in (200, 202), "Collection fixture cleanup failed") + except Exception: + if not primary_failure: + raise + print("SRE-DIAG Owned collection fixture cleanup unavailable", flush=True) diff --git a/tests/e2e/sre_authority/collection_delete_probe_test.py b/tests/e2e/sre_authority/collection_delete_probe_test.py new file mode 100644 index 000000000..1e4a173f9 --- /dev/null +++ b/tests/e2e/sre_authority/collection_delete_probe_test.py @@ -0,0 +1,109 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +import unittest +from urllib.parse import parse_qs, urlsplit +from unittest.mock import patch + +from sre_authority.collection_delete_probe import NAMESPACE_CONTROLLER, cases, dry_run_delete + + +class CollectionDeleteProofTests(unittest.TestCase): + def test_dry_run_retries_only_status_only_rv_conflicts_and_keeps_both_preconditions(self): + original = {"metadata": {"uid": "owned", "resourceVersion": "1"}, "spec": {"replicas": 0}} + updated = copy.deepcopy(original) + updated["metadata"]["resourceVersion"] = "2" + updated["status"] = {"observedGeneration": 1} + with patch("sre_authority.collection_delete_probe.request", return_value=(200, updated)), \ + patch("sre_authority.collection_delete_probe.as_tenant", + side_effect=[(409, {"reason": "Conflict"}), (200, {"kind": "Status"})]) as actor, \ + patch("sre_authority.collection_delete_probe.time.sleep"): + code, _ = dry_run_delete(1, "/collection?fieldSelector=name", "/item", original, False) + self.assertEqual(code, 200) + self.assertEqual([call.args[2]["preconditions"] for call in actor.call_args_list], [ + {"uid": "owned", "resourceVersion": "1"}, {"uid": "owned", "resourceVersion": "2"}]) + self.assertTrue(all(call.args[2]["dryRun"] == ["All"] for call in actor.call_args_list)) + self.assertEqual(original["metadata"]["resourceVersion"], "1") + + def test_dry_run_does_not_retry_replacement_content_change_or_authorization_failure(self): + original = {"metadata": {"uid": "owned", "resourceVersion": "1"}, "spec": {"replicas": 0}} + for refreshed in ( + {"metadata": {"uid": "other", "resourceVersion": "2"}, "spec": {"replicas": 0}}, + {"metadata": {"uid": "owned", "resourceVersion": "2"}, "spec": {"replicas": 1}}, + ): + with patch("sre_authority.collection_delete_probe.request", return_value=(200, refreshed)), \ + patch("sre_authority.collection_delete_probe.as_tenant", + return_value=(409, {"reason": "Conflict"})) as actor, self.assertRaises(AssertionError): + dry_run_delete(1, "/collection", "/item", original, False) + actor.assert_called_once() + with patch("sre_authority.collection_delete_probe.request") as read, \ + patch("sre_authority.collection_delete_probe.as_tenant", + return_value=(403, {"reason": "Forbidden"})) as actor: + self.assertEqual(dry_run_delete(1, "/collection", "/item", original, False)[0], 403) + actor.assert_called_once() + read.assert_not_called() + + def fixture(self): + objects, deletes, probes = {}, [], [] + def api(_port, method, path, obj=None): + parsed = urlsplit(path) + if method == "POST": + value = copy.deepcopy(obj) + value["metadata"].update(uid=value["metadata"]["name"] + "-uid", resourceVersion="1") + objects[path + "/" + value["metadata"]["name"]] = value + return 201, copy.deepcopy(value) + if method == "GET": + return 200, copy.deepcopy(objects[path]) + if parsed.query: + name = parse_qs(parsed.query)["fieldSelector"][0].removeprefix("metadata.name=") + target = parsed.path + "/" + name + self.assertEqual(obj["dryRun"], ["All"]) + else: + target = path + self.assertEqual(obj["preconditions"], { + "uid": objects[target]["metadata"]["uid"], "resourceVersion": objects[target]["metadata"]["resourceVersion"]}) + if not obj.get("dryRun"): + deletes.append(path) + del objects[path] + return 200, {"kind": "Status", "status": "Success"} + def actor(_port, path, obj, **kwargs): + self.assertEqual(kwargs, {"user": NAMESPACE_CONTROLLER, "method": "DELETE"}) + self.assertEqual(obj["dryRun"], ["All"]) + probes.append((path, obj)) + name = parse_qs(urlsplit(path).query)["fieldSelector"][0].removeprefix("metadata.name=") + policy = {"sre-api-router": "kars-sre-private-identity", + "sre-api-self-renew": "kars-sre-role-authority", + "sre": "kars-sre-consumer-authority"}.get(name) + if policy: + return 403, {"kind": "Status", "reason": "Forbidden", "message": policy + " denied do-not-publish"} + return 200, {"kind": "Status", "status": "Success"} + return objects, deletes, probes, api, actor + + def test_collections_are_body_dry_runs_with_specific_guard_denials_and_owned_cleanup(self): + objects, deletes, probes, api, actor = self.fixture() + policies = {name: {} for name in ( + "kars-sre-private-identity", "kars-sre-role-authority", "kars-sre-consumer-authority")} + with patch("sre_authority.collection_delete_probe.request", side_effect=api), \ + patch("sre_authority.collection_delete_probe.as_tenant", side_effect=actor): + results = cases(1, policies) + self.assertEqual(len(results), 9) + self.assertTrue(all(result["matched"] for result in results)) + self.assertEqual(len(probes), 6) + self.assertEqual(len(deletes), 6) + self.assertEqual(objects, {}) + self.assertNotIn("do-not-publish", json.dumps(results)) + + def test_cel_missing_name_errors_are_not_intended_authority_denials(self): + _, _, _, api, _ = self.fixture() + policies = {name: {} for name in ( + "kars-sre-private-identity", "kars-sre-role-authority", "kars-sre-consumer-authority")} + body = {"kind": "Status", "reason": "Forbidden", + "message": "kars-sre-private-identity kars-sre-role-authority kars-sre-consumer-authority " + "evaluation failed: no such key: name do-not-publish"} + with patch("sre_authority.collection_delete_probe.request", side_effect=api), \ + patch("sre_authority.collection_delete_probe.as_tenant", return_value=(403, body)): + results = cases(1, policies) + self.assertFalse(any(result["matched"] for result in results if "namespace-controller" in result["case"])) + self.assertNotIn("do-not-publish", json.dumps(results)) diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py new file mode 100644 index 000000000..582a3332c --- /dev/null +++ b/tests/e2e/sre_authority/common.py @@ -0,0 +1,719 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import base64 +import contextlib +import copy +import hashlib +import inspect +import json +import os +from pathlib import Path +import re +import socket +import signal +import ssl +import subprocess +import time + +CONTEXT = "kind-kars-e2e" +SYSTEM = "kars-system" +RUNTIME = "kars-sre" +OPERATORS = "e2e-sre-operators" +TENANT = "e2e-sre-tenant" +REGISTRATION = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical" +PRIVATE = "sre-api-router-identity" +AGENT = "sre-api-agent" +CLAIM_VERSION = "kars.azure.com/namespace-claim-version" +SOURCE_NS = "kars.azure.com/sandbox-namespace" +SOURCE_NAME = "kars.azure.com/sandbox-name" +SOURCE_UID = "kars.azure.com/sandbox-uid" +NAMESPACE_UID = "kars.azure.com/namespace-uid" +EPOCH = "kars.azure.com/sre-privacy-epoch" +OWNER = "kars.azure.com/sre-registration-uid" +PRIVACY_REVISION = "kars.azure.com/sre-privacy/v2" +FIELD_MANAGER = "kars-controller/karssandbox" +STANDIN = "kars-sandbox-e2e:dev" +POLICIES = ( + "source-authority", "registration-authority", "private-identity", + "binding-authority", "private-material", "source-retirement", + "pending-proposals", "consumer-authority", "private-mounts", + "private-workloads", "private-cronjobs", "private-connect", "role-authority", "no-legacy-tokens", +) + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def command_site(): + # Only source coordinates from our harness, never argv, frame locals, + # absolute filesystem paths or a traceback containing API response data. + frame = inspect.currentframe() + try: + while frame: + source = Path(frame.f_code.co_filename) + if source.parent == Path(__file__).parent and source.name != "common.py": + return f"{source.name}:{frame.f_code.co_name}:{frame.f_lineno}" + frame = frame.f_back + return "harness" + finally: + del frame + + +def command_error_category(stderr): + status = re.search(r"Error from server \((Forbidden|Unauthorized|Invalid|NotFound|" + r"AlreadyExists|Conflict|BadRequest|InternalError|ServiceUnavailable)\)", stderr) + if status: + return status.group(1) + if re.search(r'The CustomResourceDefinition "[^"\r\n]+" is invalid:', stderr): + return "Invalid" + text = stderr.lower() + for needle, category in (("error validating", "schema-validation"), + ("unknown flag", "cli-argument"), + ("required value", "required-field"), + ("no matches for kind", "api-discovery"), + ("conflict occurred while applying object", "server-side-apply-conflict"), + ("the server doesn't have a resource type", "api-discovery"), + ("timed out waiting", "wait-timeout")): + if needle in text: + return category + return "unclassified" + + +def authority_failure_site(root, detail): + """Report checked-in rejection coordinates, never the status detail itself.""" + if not isinstance(detail, str) or not detail: + return {"category": "unclassified"} + result = {"category": "controller-rejection"} + message = detail + status = re.fullmatch(r"(.+): Kubernetes status ([1-5][0-9]{2})", detail) + if status: + message = status.group(1) + result = {"category": "kubernetes-status", "httpStatus": int(status.group(2))} + elif detail.endswith(": Kubernetes transport/serialization failure"): + message = detail.removesuffix(": Kubernetes transport/serialization failure") + result = {"category": "kubernetes-transport"} + # Only an exact literal in our own production source can identify a site. + # Untrusted API messages, names, credentials and interpolated suffixes are + # not echoed, even if they contain a familiar rejection substring. + literal = json.dumps(message, ensure_ascii=False) + paths = [root / "controller/src/sre_authority.rs", root / "controller/src/sre_registration.rs", + root / "shared/sre_privacy.rs"] + paths += sorted((root / "controller/src/sre_authority").glob("*.rs")) + for path in paths: + if path.name.endswith("tests.rs"): + continue + for number, line in enumerate(path.read_text().splitlines(), 1): + if literal in line: + return {**result, "source": str(path.relative_to(root)), "line": number} + return {"category": "unclassified"} + + +def printed_object(output): + start = output.find("{") + require(start >= 0, "CLI omitted its JSON object") + value, _ = json.JSONDecoder().raw_decode(output[start:]) + require(isinstance(value, dict), "CLI returned a non-object JSON value") + return value + + +def enrollment_json(output): + value = printed_object(output) + require(all(key in value for key in ("controller", "sandbox", "runtimeNamespace", "legacyBindings")), + "CLI preview returned an unexpected object") + return value + + +def assert_claim(source, namespace, core, system_uid): + require(source and namespace and core, "Source/namespace/core identity is missing") + src, ns = source["metadata"], namespace["metadata"] + require(src.get("uid") and ns.get("uid") and src.get("resourceVersion") and ns.get("resourceVersion"), + "Source or namespace lacks its real API identity") + require(src.get("namespace") == SYSTEM and src.get("name") == "sre" and ns.get("name") == RUNTIME, + "Claim points outside the canonical source/runtime namespace") + require(not src.get("deletionTimestamp") and not ns.get("deletionTimestamp") and not ns.get("ownerReferences"), + "Claim is terminating or has a foreign owner") + annotations = ns.get("annotations", {}) + require(all(annotations.get(key) == value for key, value in { + CLAIM_VERSION: "v1", SOURCE_NS: SYSTEM, SOURCE_NAME: "sre", SOURCE_UID: src["uid"], + }.items()) and "kars.azure.com/namespace-prestage" not in annotations, + "Actual namespace lacks the complete current-source claim") + require(src.get("annotations", {}).get(NAMESPACE_UID) == ns["uid"], + "Actual source lacks the exact runtime-namespace UID backlink") + require(core["metadata"].get("uid") == system_uid, "Core namespace identity changed") + + +def review_args(spec, registration=None): + args = ["--sandbox-uid", spec["sandbox"]["uid"], + "--namespace-uid", spec["runtimeNamespace"]["uid"]] + for binding in spec["legacyBindings"]: + args += ["--binding", f'{binding["kind"]}/{binding.get("namespace", "")}/{binding["name"]}=' + f'{binding["uid"]}@{binding["resourceVersion"]}'] + if spec.get("legacyConsumer"): + consumer = spec["legacyConsumer"] + args += ["--consumer", f'{consumer["uid"]}@{consumer["resourceVersion"]}'] + if registration: + args += ["--registration-uid", registration["metadata"]["uid"], + "--resource-version", registration["metadata"]["resourceVersion"]] + return args + + +def assert_denial(response, label, policy=None): + # A missing CRD/resource, transport failure or malformed fixture is NOT proof. + require(response.status_code == 403, f"{label}: expected Forbidden, got HTTP {response.status_code}") + body = response.json() + require(body.get("kind") == "Status" and body.get("reason") == "Forbidden", + f"{label}: response was not a Kubernetes authorization/admission denial") + if policy: + require(policy in body.get("message", ""), f"{label}: wrong admission policy rejected the fixture") + + +class Harness: + def __init__(self, phase): + import httpx + os.umask(0o077) + self.httpx = httpx + self.root = Path(__file__).resolve().parents[3] + self.work = self.root / ".e2e-sre-authority" + self.work.mkdir(mode=0o700, exist_ok=True) + os.chmod(self.work, 0o700) + self.deadline = time.monotonic() + 650 + self.phase = phase + self.clients = {} + self.processes = [] + self.state_path = self.work / "state.json" + self.state = json.loads(self.state_path.read_text()) if self.state_path.exists() else {} + config_path = self.work / "admin.json" + if not config_path.exists(): + raw = self.run(["kubectl", "--context", CONTEXT, "--request-timeout=15s", + "config", "view", "--raw", "--minify", "-o", "json"], timeout=20) + config = json.loads(raw) + require(config["contexts"][0]["name"] == CONTEXT, "Refusing a non-Kind Kubernetes context") + config["current-context"] = CONTEXT + cluster = config["clusters"][0]["cluster"] + from urllib.parse import urlsplit + require(urlsplit(cluster["server"]).hostname in ("127.0.0.1", "::1", "localhost"), + "Refusing a non-loopback Kind API server") + self.write("admin.json", json.dumps(config)) + self.config = json.loads(config_path.read_text()) + cluster = self.config["clusters"][0]["cluster"] + self.server = cluster["server"] + self.write("api-ca.pem", base64.b64decode(cluster["certificate-authority-data"])) + admin = self.config["users"][0]["user"] + require("client-certificate-data" in admin and "client-key-data" in admin, + "Kind admin configuration must use its client certificate") + self.write("admin-cert.pem", base64.b64decode(admin["client-certificate-data"])) + self.write("admin-key.pem", base64.b64decode(admin["client-key-data"])) + + def write(self, name, value): + path = self.work / name + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "wb") as output: + output.write(value if isinstance(value, bytes) else value.encode()) + os.chmod(path, 0o600) + return path + + def save(self): + self.write("state.json", json.dumps(self.state, indent=2)) + + def run(self, args, *, data=None, user="admin", timeout=35, expected=0): + env = {**os.environ, "KARS_KUBE_CONTEXT": CONTEXT, "NO_COLOR": "1", "FORCE_COLOR": "0"} + config = self.work / f"{user}.json" + if config.exists(): + env["KUBECONFIG"] = str(config) + else: + require(user == "admin", "Probe principal has no isolated kubeconfig") + remaining = self.deadline - time.monotonic() + require(remaining > 0, "SRE acceptance phase exceeded its bounded deadline") + process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, cwd=self.root, + env=env, start_new_session=True) + try: + stdout, stderr = process.communicate(input=data, timeout=min(timeout, remaining)) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGTERM) + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=5) + raise AssertionError(f"Command exceeded its bounded timeout at {command_site()}") from None + result = subprocess.CompletedProcess(args, process.returncode, stdout, stderr) + if expected is not None: + # Never echo command output or argv: token/Secret reads are captured. + require(result.returncode == expected, + f"Command failed during {self.phase} at {command_site()}; " + f"exit={result.returncode}; category={command_error_category(stderr)}") + return result if expected is None else result.stdout + + def k(self, *args, data=None, user="admin", timeout=35, expected=0): + request_timeout = max(1, int(min(timeout - 2, self.deadline - time.monotonic()))) + return self.run(["kubectl", "--context", CONTEXT, f"--request-timeout={request_timeout}s", *args], + data=data, user=user, timeout=timeout, expected=expected) + + def cli(self, *args, user="admin", expected=0, timeout=240): + return self.run(["node", str(self.root / "cli/dist/index.js"), "sre", *args, + "--context", CONTEXT, "--namespace", SYSTEM, "--release", "kars"], + user=user, expected=expected, timeout=timeout) + + def client(self, user="admin"): + if user not in self.clients: + context = ssl.create_default_context(cafile=str(self.work / "api-ca.pem")) + headers = {"Accept": "application/json"} + if user == "admin": + context.load_cert_chain(str(self.work / "admin-cert.pem"), str(self.work / "admin-key.pem")) + else: + # Deliberately separate SSLContext: wrong-token probes MUST NOT + # retain an admin certificate, exec plugin or auth-provider. + config = json.loads((self.work / f"{user}.json").read_text()) + account = config["users"][0]["user"] + require(set(account) == {"token"}, "Non-admin probe retained privileged kubeconfig auth") + headers["Authorization"] = f'Bearer {account["token"]}' + self.clients[user] = self.httpx.Client(base_url=self.server, verify=context, + headers=headers, timeout=15, trust_env=False) + return self.clients[user] + + def api(self, method, path, *, body=None, user="admin", status=None): + headers = {"Content-Type": "application/merge-patch+json"} if method == "PATCH" else None + response = self.client(user).request(method, path, json=body, headers=headers) + if status is not None: + require(response.status_code in (status if isinstance(status, tuple) else (status,)), + f"{method} {path.split('?')[0]}: HTTP {response.status_code}, expected {status}") + return response + + def get(self, kind, name, namespace=None): + args = ["get", kind, name, "--ignore-not-found", "-o", "json"] + if namespace: + args += ["-n", namespace] + raw = self.k(*args) + return json.loads(raw) if raw.strip() else None + + def create(self, obj, manager=None): + args = ["create", "-f", "-", "-o", "json"] + if manager: + args += ["--field-manager", manager] + return json.loads(self.k(*args, data=json.dumps(obj))) + + def token_identity(self, name, namespace, account): + token = self.k("create", "token", account, "-n", namespace, "--duration=1h").strip() + require(token, "TokenRequest returned no credential") + return self.token_config(name, token) + + def token_config(self, name, token): + cluster = self.config["clusters"][0] + self.write(f"{name}.json", json.dumps({ + "apiVersion": "v1", "kind": "Config", "current-context": CONTEXT, + "clusters": [cluster], "users": [{"name": name, "user": {"token": token}}], + "contexts": [{"name": CONTEXT, "context": {"cluster": cluster["name"], "user": name}}], + })) + old = self.clients.pop(name, None) + if old: + old.close() + + def poll(self, label, predicate, seconds=150, interval=1): + end = min(self.deadline, time.monotonic() + seconds) + while time.monotonic() < end: + value = predicate() + if value: + return value + time.sleep(interval) + raise AssertionError(f"{label} did not converge before its deadline") + + def wait_ready(self): + def ready(): + reg = self.get("karssreregistrations.kars.azure.com", "canonical") + if not reg: + return False + status = reg.get("status", {}) + require(not (status.get("phase") == "Blocked" and status.get("observedGeneration") == reg["metadata"]["generation"]), + "SRE authority reported Blocked; see sanitized status diagnostics") + return reg if (status.get("phase") == "Ready" + and status.get("observedGeneration") == reg["metadata"]["generation"] + and status.get("privacyRevision") == PRIVACY_REVISION + and status.get("legacySecretAccessDenied") is True) else False + return self.poll("SRE Ready", ready, seconds=240) + + def namespace_delete(self, name, uid): + obj = self.get("namespace", name) + require(obj and obj["metadata"]["uid"] == uid, "Fixture namespace changed before cleanup") + self.api("DELETE", f"/api/v1/namespaces/{name}", body={ + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": obj["metadata"]["resourceVersion"]}, + }, status=(200, 202)) + self.poll(f"namespace {name} cleanup", lambda: self.get("namespace", name) is None, seconds=120) + + @contextlib.contextmanager + def port_forward(self, pod): + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + log = self.work / "port-forward.log" + with log.open("w") as output: + process = subprocess.Popen(["kubectl", "--context", CONTEXT, "--request-timeout=20s", + "-n", RUNTIME, "port-forward", f"pod/{pod}", f"{port}:9446", "--address=127.0.0.1"], + cwd=self.root, env={**os.environ, "KUBECONFIG": str(self.work / "registrar.json")}, + stdout=output, stderr=output) + self.processes.append(process) + try: + self.poll("private TLS port-forward", lambda: process.poll() is None + and log.exists() and "Forwarding from" in log.read_text(), seconds=25) + yield port + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + self.processes.remove(process) + + def passed(self, message): + print(f"SRE-PASS {message}", flush=True) + + def diagnostics(self): + from .bootstrap_diagnostics import failure_facts + self.deadline = max(self.deadline, time.monotonic() + 90) + # Status and identities only; never dump Secret bodies or whole Pods. + for kind, name, namespace in [("karssreregistrations.kars.azure.com", "canonical", None), + ("karssandbox", "sre", SYSTEM), ("deployment", "sre", RUNTIME), + ("namespace", RUNTIME, None)]: + try: + obj = self.get(kind, name, namespace) + if obj: + status = obj.get("status", {}) + print("SRE-DIAG", json.dumps({"kind": kind, "name": name, + "uid": obj["metadata"].get("uid"), "phase": status.get("phase"), + "observedGeneration": status.get("observedGeneration"), + "generation": obj["metadata"].get("generation"), + "availableReplicas": status.get("availableReplicas"), + **({"authorityFailure": authority_failure_site(self.root, status.get("detail"))} + if kind == "karssreregistrations.kars.azure.com" and status.get("phase") == "Blocked" else {}), + "conditions": [{"type": condition.get("type"), "status": condition.get("status"), + "reason": condition.get("reason"), + **failure_facts(condition.get("message"), + {f"kars-sre-{name}": {} for name in POLICIES})} + for condition in status.get("conditions", [])]}), flush=True) + except Exception: + print(f"SRE-DIAG {kind}/{name} unavailable", flush=True) + try: + pods = json.loads(self.k("get", "pods", "-n", RUNTIME, "-o", "json", timeout=10))["items"] + for pod in pods: + status = pod.get("status", {}) + containers = status.get("initContainerStatuses", []) + status.get("containerStatuses", []) + print("SRE-DIAG", json.dumps({"kind": "Pod", "name": pod["metadata"]["name"], + "uid": pod["metadata"]["uid"], "phase": status.get("phase"), + "containers": [{"name": container["name"], "ready": container.get("ready"), + "restartCount": container.get("restartCount"), + "state": {kind: {key: value.get(key) for key in ("reason", "exitCode") if key in value} + for kind, value in container.get("state", {}).items()}} + for container in containers]}), flush=True) + router = next((container for container in status.get("containerStatuses", []) + if container["name"] == "inference-router"), None) + if router and not router.get("ready") and "running" in router.get("state", {}): + self.readiness_diagnostics(pod) + elif router and router.get("ready"): + from .bootstrap_diagnostics import router_readiness_facts + try: + logs = self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", + "--tail=150", timeout=10) + print("SRE-DIAG", json.dumps({"kind": "RouterAuthorityLog", "podUid": pod["metadata"]["uid"], + "authorityChecks": router_readiness_facts(logs)}), flush=True) + except Exception: + print("SRE-DIAG Ready router authority log unavailable", flush=True) + except Exception: + print("SRE-DIAG runtime Pod status unavailable", flush=True) + + def readiness_diagnostics(self, pod): + from .bootstrap_diagnostics import probe_command_result, router_log_summary, router_readiness_facts + facts = {"kind": "RouterReadiness", "podUid": pod["metadata"]["uid"]} + router = next(item for item in pod["spec"]["containers"] if item["name"] == "inference-router") + facts["privateApiEnabled"] = any(entry.get("name") == "KARS_SRE_API_ENABLED" + and entry.get("value") == "true" for entry in router.get("env", [])) + facts["routerUid1001"] = router.get("securityContext", {}).get("runAsUser") == 1001 + facts["expectedImage"] = router.get("image") == "kars-inference-router:e2e" + facts["progressLoggingEnabled"] = any(entry.get("name") == "RUST_LOG" + and "inference_router=debug" in entry.get("value", "") + for entry in router.get("env", [])) + try: + facts["apiConnectivity"] = self.connectivity_diagnostics(pod) + except Exception as error: + facts["connectivityDiagnosticError"] = type(error).__name__ + try: + source = self.get("karssandbox", "sre", SYSTEM) + policy = self.get("networkpolicy", "sandbox-policy", RUNTIME) + service = self.get("service", "kubernetes", "default") + service_ip = service["spec"]["clusterIP"] + rules = policy.get("spec", {}).get("egress", []) if policy else [] + facts["sourceLabelSre"] = source.get("metadata", {}).get("labels", {}).get("kars.azure.com/role") == "sre" + facts["explicitApiServiceRule"] = any( + any(peer.get("ipBlock", {}).get("cidr") == f"{service_ip}/32" for peer in rule.get("to", [])) + and any(port.get("port") == 443 and port.get("protocol", "TCP") == "TCP" + for port in rule.get("ports", [])) for rule in rules) + agents = json.loads(self.k("get", "daemonsets", "-n", "kube-system", "-o", "json"))["items"] + facts["knownNetworkAgents"] = sorted(agent["metadata"]["name"] for agent in agents + if agent["metadata"]["name"] in ("kindnet", "cilium", "calico-node", "antrea-agent", "kube-flannel-ds")) + facts["minimalKindnetImage"] = any( + container.get("image", "").removeprefix("docker.io/").startswith("kindest/kindnetd:") + for agent in agents if agent["metadata"]["name"] == "kindnet" + for container in agent.get("spec", {}).get("template", {}).get("spec", {}).get("containers", [])) + except Exception as error: + facts["networkPolicyDiagnosticError"] = type(error).__name__ + try: + facts["firewall"] = self.firewall_diagnostics(pod) + except Exception as error: + facts["firewallDiagnosticError"] = type(error).__name__ + try: + for label, executable in (("configuredCommand", "kars-inference-router"), + ("absoluteCommand", "/usr/local/bin/kars-inference-router")): + result = self.k("exec", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", + "--", executable, "sre-ready", expected=None, timeout=10) + facts[label] = probe_command_result(result.returncode, result.stdout + result.stderr) + ca = self.k("exec", "-n", RUNTIME, pod["metadata"]["name"], "-c", "agent", + "--", "cat", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", timeout=10) + context = ssl.create_default_context(cadata=ca) + with self.port_forward(pod["metadata"]["name"]) as port, \ + self.httpx.Client(verify=context, timeout=25, trust_env=False) as client: + response = client.get(f"https://127.0.0.1:{port}/readyz") + facts["verifiedLoopbackTlsStatus"] = response.status_code + except Exception as error: + facts["diagnosticError"] = type(error).__name__ + try: + logs = self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", + "--tail=150", timeout=10) + facts["authorityChecks"] = router_readiness_facts(logs) + facts["logSummary"] = router_log_summary(logs, self.root) + except Exception: + facts["authorityChecksUnavailable"] = True + print("SRE-DIAG", json.dumps(facts), flush=True) + + def firewall_diagnostics(self, pod): + from .bootstrap_diagnostics import firewall_summary + node = pod["spec"].get("nodeName") + require(node in ("kars-e2e-worker", "kars-e2e-control-plane") + and not pod["spec"].get("hostNetwork"), "Firewall diagnostic requires the owned Kind Pod network") + cluster = self.run(["docker", "inspect", node, "--format", + '{{index .Config.Labels "io.x-k8s.kind.cluster"}}'], timeout=10).strip() + require(cluster == "kars-e2e", "Firewall diagnostic refuses a foreign node") + items = json.loads(self.run(["docker", "exec", node, "crictl", "pods", + "--label", f'io.kubernetes.pod.uid={pod["metadata"]["uid"]}', + "-o", "json"], timeout=10))["items"] + matches = [item for item in items if item.get("metadata", {}).get("uid") == pod["metadata"]["uid"] + and item["metadata"].get("namespace") == RUNTIME and item.get("state") == "SANDBOX_READY"] + require(len(matches) == 1 and re.fullmatch(r"[0-9a-f]{64}", matches[0]["id"]), + "Firewall diagnostic could not identify the exact live Pod sandbox") + sandbox = matches[0]["id"] + def identity(): + obj = json.loads(self.run(["docker", "exec", node, "crictl", "inspectp", "-o", "json", sandbox], timeout=10)) + require(obj.get("status", {}).get("metadata", {}).get("uid") == pod["metadata"]["uid"], + "Firewall diagnostic sandbox UID changed") + pid = obj.get("info", {}).get("pid") + require(type(pid) is int and 0 < pid < 2**31, "Firewall diagnostic sandbox PID is invalid") + return pid + pid = identity() + facts = {} + for backend in ("nft", "legacy"): + output = self.run(["docker", "exec", node, "nsenter", "--target", str(pid), "--net", "--", + f"iptables-{backend}-save", "-c"], timeout=10, expected=None) + facts[backend] = firewall_summary(output.stdout) if output.returncode == 0 else {"available": False} + require(identity() == pid, "Firewall diagnostic sandbox changed during the read") + return facts + + def connectivity_diagnostics(self, pod): + import ipaddress + service = self.get("service", "kubernetes", "default") + endpoint = self.get("endpoints", "kubernetes", "default") + service_ip = service["spec"]["clusterIP"] + service_port = next(port["port"] for port in service["spec"]["ports"] if port["name"] == "https") + subset = endpoint["subsets"][0] + endpoint_ip = subset["addresses"][0]["ip"] + endpoint_port = next(port["port"] for port in subset["ports"] if port["name"] == "https") + require(all(ipaddress.ip_address(address).is_private for address in (service_ip, endpoint_ip)) + and all(type(port) is int and 0 < port < 65536 for port in (service_port, endpoint_port)), + "Connectivity diagnostic refuses non-private or invalid API targets") + current = self.get("pod", pod["metadata"]["name"], RUNTIME) + require(current["metadata"]["uid"] == pod["metadata"]["uid"], + "Diagnostic Pod changed before connectivity inspection") + require(current["spec"].get("automountServiceAccountToken") is False + and current["spec"].get("shareProcessNamespace") is not True, + "Connectivity diagnostic requires isolated processes and no ambient token") + name = "sre-e2e-network-diagnostic" + existing = current["spec"].get("ephemeralContainers", []) + require(not any(item["name"] == name for item in existing), "Diagnostic container name is already occupied") + script = """ +if ! command -v timeout >/dev/null || ! command -v bash >/dev/null || ! command -v id >/dev/null; then + printf '{"available":false}\\n'; exit 0 +fi +uid=false +[ "$(id -u)" = 1001 ] && uid=true +timeout 6 bash -c 'exec 3<>/dev/tcp/"$1"/"$2"' sre-tcp "$1" "$2" >/dev/null 2>&1 +service=$? +timeout 6 bash -c 'exec 3<>/dev/tcp/"$1"/"$2"' sre-tcp "$3" "$4" >/dev/null 2>&1 +endpoint=$? +printf '{"available":true,"uidMatches1001":%s,"serviceExit":%s,"endpointExit":%s}\\n' "$uid" "$service" "$endpoint" +""" + probe = {"name": name, "image": STANDIN, "imagePullPolicy": "IfNotPresent", + "command": ["/bin/sh", "-c", script, "sre-connectivity", service_ip, str(service_port), + endpoint_ip, str(endpoint_port)], + "securityContext": {"runAsUser": 1001, "runAsNonRoot": True, + "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, + "capabilities": {"drop": ["ALL"]}}} + self.api("PATCH", f"/api/v1/namespaces/{RUNTIME}/pods/{pod['metadata']['name']}/ephemeralcontainers", + body={"metadata": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}, + "spec": {"ephemeralContainers": existing + [probe]}}, status=200) + def completed(): + current = self.get("pod", pod["metadata"]["name"], RUNTIME) + require(current and current["metadata"]["uid"] == pod["metadata"]["uid"], + "Diagnostic Pod changed during connectivity inspection") + return any(item.get("name") == name and "terminated" in item.get("state", {}) + for item in current.get("status", {}).get("ephemeralContainerStatuses", [])) + self.poll("bounded UID-1001 API TCP probes", completed, seconds=20, interval=0.5) + result = self.parse_connectivity_result(self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", name, "--tail=5")) + try: + result["sameNodeControl"] = self.control_connectivity_diagnostics(pod, probe) + except Exception as error: + result["controlDiagnosticError"] = type(error).__name__ + result["guardControls"] = {} + for variant in ("full", "filter-only", "legacy-full"): + try: + result["guardControls"][variant] = self.control_connectivity_diagnostics(pod, probe, variant) + except Exception as error: + result["guardControls"][variant] = {"diagnosticError": type(error).__name__} + try: + result["policyControl"] = self.policy_connectivity_diagnostics(pod, probe) + except Exception as error: + result["policyControl"] = {"diagnosticError": type(error).__name__} + try: + result["denyAllPolicyControl"] = self.policy_connectivity_diagnostics(pod, probe, deny_all=True) + except Exception as error: + result["denyAllPolicyControl"] = {"diagnosticError": type(error).__name__} + try: + result["apiEndpointPolicyControl"] = self.policy_connectivity_diagnostics(pod, probe, api_endpoint=True) + except Exception as error: + result["apiEndpointPolicyControl"] = {"diagnosticError": type(error).__name__} + return result + + @staticmethod + def parse_connectivity_result(output, uid=1001): + require(uid in (1000, 1001), "Unsupported connectivity probe UID") + uid_key = f"uidMatches{uid}" + result = json.loads(output) + require(isinstance(result, dict) and type(result.get("available")) is bool + and set(result) == ({"available", uid_key, "serviceExit", "endpointExit"} + if result["available"] else {"available"}) + and (not result["available"] or type(result.get(uid_key)) is bool) + and all(type(value) is int and 0 <= value <= 255 + for key, value in result.items() if key not in ("available", uid_key)), + "Connectivity diagnostic produced an unexpected result") + return result + + def control_connectivity_diagnostics(self, pod, container, variant=None, *, policy=False, deny_all=False, api_endpoint=False): + from .network_diagnostics import guard_variant + suffix = "-endpoint" if api_endpoint else "-deny-all" if deny_all else "-policy" if policy else f"-{variant}" if variant else "" + name = "sre-e2e-api-connectivity" + suffix + require(pod["spec"].get("nodeName"), "Connectivity comparison requires the actual sandbox node") + spec = {"nodeName": pod["spec"]["nodeName"], "automountServiceAccountToken": False, + "restartPolicy": "Never", "securityContext": copy.deepcopy(pod["spec"].get("securityContext", {})), + "containers": [copy.deepcopy(container)]} + if variant or api_endpoint: + spec["initContainers"] = [guard_variant(self.root, pod, variant or "full")] + if api_endpoint: + agent = copy.deepcopy(container) + agent["name"] = "agent-network" + agent["securityContext"]["runAsUser"] = 1000 + script = agent["command"][2] + require(script.count("= 1001") == 1 and script.count("uidMatches1001") == 1, + "Agent network comparison requires the exact diagnostic script") + agent["command"][2] = script.replace("= 1001", "= 1000").replace("uidMatches1001", "uidMatches1000") + spec["containers"].append(agent) + metadata = {"name": name, "namespace": TENANT} + if policy: + value = "endpoint" if api_endpoint else "deny-all" if deny_all else "true" + metadata["labels"] = {"kars.azure.com/e2e-policy-control": value} + for probe in spec["containers"]: + probe["command"][2] = "sleep 10\n" + probe["command"][2] + created = self.create({"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}) + uid = created["metadata"]["uid"] + try: + def completed(): + current = self.get("pod", name, TENANT) + require(current and current["metadata"]["uid"] == uid, "Connectivity control Pod was replaced") + status = current.get("status", {}) + return status.get("phase") in ("Succeeded", "Failed") or any( + item.get("state", {}).get("terminated", {}).get("exitCode", 0) != 0 + for item in status.get("initContainerStatuses", [])) + self.poll("same-node credential-free TCP comparison", completed, seconds=35, interval=0.5) + current = self.get("pod", name, TENANT) + for item in current.get("status", {}).get("initContainerStatuses", []): + code = item.get("state", {}).get("terminated", {}).get("exitCode", 0) + if code != 0: + return {"available": False, "initExit": code} + result = self.parse_connectivity_result( + self.k("logs", "-n", TENANT, name, "-c", container["name"], "--tail=5")) + if api_endpoint: + result["agentUid1000"] = self.parse_connectivity_result( + self.k("logs", "-n", TENANT, name, "-c", "agent-network", "--tail=5"), uid=1000) + return result + finally: + try: + current = self.get("pod", name, TENANT) + require(current and current["metadata"]["uid"] == uid, "Connectivity control cleanup identity changed") + self.api("DELETE", f"/api/v1/namespaces/{TENANT}/pods/{name}", body={ + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, + status=(200, 202)) + except Exception: + print("SRE-DIAG owned connectivity control cleanup unavailable", flush=True) + + def policy_connectivity_diagnostics(self, pod, container, *, deny_all=False, api_endpoint=False): + import ipaddress + source = self.get("networkpolicy", "sandbox-policy", RUNTIME) + require(source is not None, "SRE network policy is unavailable for a read-only comparison") + spec = {"policyTypes": ["Egress"], "egress": []} if deny_all else copy.deepcopy(source["spec"]) + if api_endpoint: + endpoint = self.get("endpoints", "kubernetes", "default") + rules = [] + for subset in endpoint.get("subsets", []): + for address in subset.get("addresses", []): + ip = ipaddress.ip_address(address["ip"]) + require(ip.is_private and not ip.is_loopback, "Unexpected diagnostic API endpoint address") + for port in subset.get("ports", []): + if port.get("name") == "https" and port.get("protocol", "TCP") == "TCP": + require(type(port["port"]) is int and 0 < port["port"] < 65536, + "Invalid diagnostic API endpoint port") + rules.append({"to": [{"ipBlock": {"cidr": f"{ip}/{ip.max_prefixlen}"}}], + "ports": [{"protocol": "TCP", "port": port["port"]}]}) + require(0 < len(rules) <= 16, "Diagnostic API endpoint inventory is empty or unbounded") + spec["egress"] = spec.get("egress", []) + rules + value = "endpoint" if api_endpoint else "deny-all" if deny_all else "true" + spec["podSelector"] = {"matchLabels": {"kars.azure.com/e2e-policy-control": value}} + name = "sre-e2e-policy-control" + ("-endpoint" if api_endpoint else "-deny-all" if deny_all else "") + created = self.create({"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": name, "namespace": TENANT}, "spec": spec}) + uid = created["metadata"]["uid"] + try: + return self.control_connectivity_diagnostics(pod, container, policy=True, + deny_all=deny_all, api_endpoint=api_endpoint) + finally: + try: + current = self.get("networkpolicy", name, TENANT) + require(current and current["metadata"]["uid"] == uid, "Connectivity policy cleanup identity changed") + self.api("DELETE", f"/apis/networking.k8s.io/v1/namespaces/{TENANT}/networkpolicies/{name}", body={ + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, + status=(200, 202)) + except Exception: + print("SRE-DIAG owned connectivity policy cleanup unavailable", flush=True) + + def close(self): + for process in self.processes: + process.terminate() + for client in self.clients.values(): + client.close() + + +def fingerprint(secret, key="control-token"): + return hashlib.sha256(base64.b64decode(secret["data"][key])).hexdigest() diff --git a/tests/e2e/sre_authority/controller_update_probe.py b/tests/e2e/sre_authority/controller_update_probe.py new file mode 100644 index 000000000..536cf610f --- /dev/null +++ b/tests/e2e/sre_authority/controller_update_probe.py @@ -0,0 +1,98 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Actual UPDATE/owner-reference checks for the narrow ReplicaSet capability.""" + +import copy + +from .bootstrap_cases import DEPLOYMENT_CONTROLLER, USER, as_tenant +from .bootstrap_diagnostics import api_result +from .common import require +from .registration_schema import request + + +def cases(port, policies, parent): + namespace = "kars-sre" + parent_path = f"/apis/apps/v1/namespaces/{namespace}/deployments/{parent['metadata']['name']}" + code, current = request(port, "GET", parent_path) + require(code == 200 and current["metadata"]["uid"] == parent["metadata"]["uid"] + and current["spec"] == parent["spec"], "Approved private parent changed before update proof") + owner = {"apiVersion": "apps/v1", "kind": "Deployment", "name": parent["metadata"]["name"], + "uid": parent["metadata"]["uid"], "controller": True} + private_pod = copy.deepcopy(parent["spec"]["template"]["spec"]) + ordinary_pod = copy.deepcopy(private_pod) + ordinary_pod.pop("volumes", None) + for container in ordinary_pod["containers"]: + container.pop("volumeMounts", None) + paths = "/apis/apps/v1/namespaces/kars-sre/replicasets" + fixtures, reports = [], [] + primary_failure = False + try: + for suffix, template in (("ordinary", ordinary_pod), ("private", private_pod)): + name = f"e2e-update-{suffix}" + selector = {"app": name} + obj = {"apiVersion": "apps/v1", "kind": "ReplicaSet", + "metadata": {"name": name, "namespace": namespace}, + "spec": {"replicas": 0, "selector": {"matchLabels": selector}, + "template": {"metadata": {"labels": selector}, "spec": template}}} + code, created = request(port, "POST", paths, obj) + require(code == 201 and created.get("metadata", {}).get("uid"), + "Owned update fixture CREATE failed") + fixtures.append((f"{paths}/{name}", created)) + ordinary_path, ordinary = fixtures[0] + private_path, private = fixtures[1] + probes = [ + ("tenant-private-rs-forged-parent-create", USER, "POST", paths, private, True, True, 403), + ("tenant-ordinary-rs-update", USER, "PUT", ordinary_path, ordinary, False, False, 200), + ("tenant-private-rs-forged-parent-update", USER, "PUT", ordinary_path, ordinary, True, True, 403), + ("tenant-owned-private-rs-ownerref-update", USER, "PUT", private_path, private, True, True, 403), + ("deployment-controller-private-rs-update", DEPLOYMENT_CONTROLLER, "PUT", private_path, private, True, True, 200), + ("tenant-private-parent-update", USER, "PUT", parent_path, parent, True, False, 403), + ] + for label, principal, method, path, original, is_private, reference, expected in probes: + if method == "PUT": + code, obj = request(port, "GET", path) + require(code == 200 and obj["metadata"]["uid"] == original["metadata"]["uid"], + "Update proof target UID changed") + obj = copy.deepcopy(obj) + else: + obj = copy.deepcopy(original) + obj["metadata"] = {"name": "e2e-forged-private-create", "namespace": namespace} + obj.pop("status", None) + if is_private: + obj["spec"]["template"]["spec"] = copy.deepcopy(private_pod) + if reference: + obj["metadata"]["ownerReferences"] = [owner] + obj["spec"]["template"].setdefault("metadata", {}).setdefault("annotations", {})["e2e-update-proof"] = "true" + code, body = as_tenant(port, path + "?dryRun=All", obj, user=principal, method=method) + result = api_result(code, body, policies) + result.update({"case": label, "expectedStatus": expected, + "matched": code == expected and (expected != 403 + or "kars-sre-private-workloads" in result.get("policies", [])) + and (expected == 403 or body.get("kind") == obj["kind"] + and body.get("metadata", {}).get("uid") == original["metadata"]["uid"])}) + reports.append(result) + for path, original in fixtures + [(parent_path, parent)]: + code, current = request(port, "GET", path) + require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"] + and current["spec"] == original["spec"] + and current["metadata"].get("ownerReferences") == original["metadata"].get("ownerReferences"), + "Owner/update dry-run changed a real workload") + return reports + except BaseException: + primary_failure = True + raise + finally: + for path, original in reversed(fixtures): + try: + code, current = request(port, "GET", path) + require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"], + "Update fixture cleanup identity changed") + code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}}) + require(code in (200, 202), "Owned update fixture cleanup failed") + except Exception: + if not primary_failure: + raise + print("SRE-DIAG Owned update fixture cleanup unavailable", flush=True) diff --git a/tests/e2e/sre_authority/controller_update_probe_test.py b/tests/e2e/sre_authority/controller_update_probe_test.py new file mode 100644 index 000000000..27e0d0346 --- /dev/null +++ b/tests/e2e/sre_authority/controller_update_probe_test.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +import unittest +from unittest.mock import patch + +from sre_authority.bootstrap_cases import DEPLOYMENT_CONTROLLER, USER +from sre_authority.controller_update_probe import cases + + +class ControllerUpdateProofTests(unittest.TestCase): + def fixture(self): + parent = {"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": "approved-parent", "namespace": "kars-sre", "uid": "parent", "resourceVersion": "1"}, + "spec": {"replicas": 0, "template": {"spec": { + "containers": [{"name": "probe", "image": "registry.invalid/probe:never", + "volumeMounts": [{"name": "private", "mountPath": "/private"}]}], + "volumes": [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}]}}}} + path = "/apis/apps/v1/namespaces/kars-sre/deployments/approved-parent" + objects = {path: copy.deepcopy(parent)} + calls = [] + def api(_port, method, path, obj=None): + calls.append((method, path, copy.deepcopy(obj))) + if method == "GET": + return 200, copy.deepcopy(objects[path]) + if method == "POST": + created = copy.deepcopy(obj) + created["metadata"].update(uid=created["metadata"]["name"], resourceVersion="1") + objects[path + "/" + created["metadata"]["name"]] = created + return 201, copy.deepcopy(created) + if method == "DELETE": + self.assertEqual(obj["preconditions"], { + "uid": objects[path]["metadata"]["uid"], "resourceVersion": objects[path]["metadata"]["resourceVersion"]}) + del objects[path] + return 200, {"kind": "Status"} + raise AssertionError("Unexpected API mutation") + return parent, objects, calls, api + + def test_update_ownerrefs_use_actual_parent_and_never_persist_dry_runs(self): + parent, objects, operations, api = self.fixture() + outcomes = iter([403, 200, 403, 403, 200, 403]) + probes = [] + def actor(_port, path, obj, **kwargs): + probes.append((path, copy.deepcopy(obj), kwargs)) + code = next(outcomes) + return code, (copy.deepcopy(obj) if code == 200 else { + "kind": "Status", "reason": "Forbidden", + "message": "kars-sre-private-workloads denied do-not-publish"}) + with patch("sre_authority.controller_update_probe.request", side_effect=api), \ + patch("sre_authority.controller_update_probe.as_tenant", side_effect=actor): + result = cases(1, {"kars-sre-private-workloads": {}}, parent) + self.assertTrue(all(case["matched"] for case in result)) + self.assertNotIn("do-not-publish", json.dumps(result)) + self.assertTrue(all(path.endswith("?dryRun=All") for path, _, _ in probes)) + self.assertEqual(probes[0][2], {"user": USER, "method": "POST"}) + self.assertEqual(probes[4][2], {"user": DEPLOYMENT_CONTROLLER, "method": "PUT"}) + self.assertEqual(probes[5][1]["kind"], "Deployment") + for index in (0, 2, 3, 4): + self.assertEqual(probes[index][1]["metadata"]["ownerReferences"], [{ + "apiVersion": "apps/v1", "kind": "Deployment", "name": "approved-parent", + "uid": "parent", "controller": True}]) + self.assertIn("volumes", probes[index][1]["spec"]["template"]["spec"]) + self.assertNotIn("volumes", probes[1][1]["spec"]["template"]["spec"]) + self.assertEqual(len(objects), 1) + self.assertEqual(sum(method == "DELETE" for method, _, _ in operations), 2) + + def test_unrelated_denial_or_missing_endpoint_is_not_an_admission_pass(self): + for code in (403, 404, 422): + parent, _, _, api = self.fixture() + with patch("sre_authority.controller_update_probe.request", side_effect=api), \ + patch("sre_authority.controller_update_probe.as_tenant", + return_value=(code, {"kind": "Status", "reason": "Forbidden", "message": "unrelated"})): + result = cases(1, {"kars-sre-private-workloads": {}}, parent) + self.assertFalse(any(case["matched"] for case in result)) + + def test_replaced_or_changed_approved_parent_stops_before_fixture_creation(self): + parent, _, _, _ = self.fixture() + for changed in ({"metadata": {"uid": "other"}, "spec": parent["spec"]}, + {"metadata": parent["metadata"], "spec": {}}): + with patch("sre_authority.controller_update_probe.request", return_value=(200, changed)) as api, \ + self.assertRaises(AssertionError): + cases(1, {}, parent) + api.assert_called_once() diff --git a/tests/e2e/sre_authority/credential_paths.py b/tests/e2e/sre_authority/credential_paths.py new file mode 100644 index 000000000..cdc5c6585 --- /dev/null +++ b/tests/e2e/sre_authority/credential_paths.py @@ -0,0 +1,341 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import base64 +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +import copy +import json +import threading +import time + +from .common import AGENT, OWNER, PRIVATE, REGISTRATION, RUNTIME, SYSTEM, assert_denial, require + +WATCH_BINDING = "e2e-sre-legacy-watch-only" +TOKEN_ALIAS = "e2e-prestaged-router-token" +TOKEN_TYPE = "kubernetes.io/service-account-token" +SA_NAME = "kubernetes.io/service-account.name" +SA_UID = "kubernetes.io/service-account.uid" +SYNTHETIC_TOKEN = "kind-synthetic-not-a-jwt-never-authenticates" +WATCH_MARKER = "kind-dummy-secret-watch-proof" + + +def synthetic_token_secret(h, name, account): + metadata = account["metadata"] + require(metadata.get("uid") and metadata.get("namespace") == RUNTIME + and not metadata.get("deletionTimestamp"), "Synthetic token fixture requires a live ServiceAccount UID") + ca = h.poll("public namespace CA for synthetic token fixture", + lambda: h.get("configmap", "kube-root-ca.crt", RUNTIME), seconds=30) + ca = ca.get("data", {}).get("ca.crt", "") + require(ca.startswith("-----BEGIN CERTIFICATE-----"), "Synthetic token fixture lacks the public cluster CA") + # A matching live SA UID and complete data prevent native TokenController + # garbage collection and token generation. Never authenticate with this data. + return {"apiVersion": "v1", "kind": "Secret", "type": TOKEN_TYPE, + "metadata": {"name": name, "namespace": RUNTIME, + "annotations": {SA_NAME: metadata["name"], SA_UID: metadata["uid"]}}, + "data": {key: base64.b64encode(value.encode()).decode() for key, value in { + "token": SYNTHETIC_TOKEN, "ca.crt": ca, "namespace": RUNTIME}.items()}} + + +def assert_secret_unchanged(current, before): + require(current is not None, "Token Secret fixture disappeared; absence is not denial proof") + require(all(current["metadata"].get(key) == before["metadata"].get(key) + for key in ("name", "namespace", "uid", "resourceVersion", "annotations", "labels", + "ownerReferences", "finalizers", "deletionTimestamp")) + and current.get("type") == before.get("type") + and current.get("data", {}) == before.get("data", {}), + "Token Secret fixture was replaced, mutated or populated") + if current.get("type") == TOKEN_TYPE: + require(current.get("data", {}).get("token") == base64.b64encode(SYNTHETIC_TOKEN.encode()).decode(), + "Token Secret fixture no longer contains the exact synthetic non-JWT") + + +def assert_token_type_immutable(response, label): + """Secret type validation precedes VAP; this is not admission-policy proof.""" + require(response.status_code == 422, + f"{label}: expected immutable Secret type rejection, got HTTP {response.status_code}") + body = response.json() + require(body.get("kind") == "Status" and body.get("reason") == "Invalid" + and any(cause.get("field") == "type" + and cause.get("reason") == "FieldValueInvalid" + and "field is immutable" in cause.get("message", "") + for cause in body.get("details", {}).get("causes", [])), + f"{label}: missing the specific immutable type validation cause") + + +def seed_privacy_gaps(h): + require(h.get("serviceaccount", "sre-api-router", RUNTIME) is None, + "Legacy fixtures require an unoccupied reserved ServiceAccount name") + account = h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "sre-api-router", "namespace": RUNTIME}, + "automountServiceAccountToken": False}) + h.state["prestaged_account"] = account + h.state["prestaged_account_cleaned"] = False + alias = h.create(synthetic_token_secret(h, TOKEN_ALIAS, account)) + assert_secret_unchanged(h.get("secret", TOKEN_ALIAS, RUNTIME), alias) + h.state["prestaged_alias"] = alias + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": WATCH_BINDING}, + "rules": [{"apiGroups": [""], "resources": ["secrets"], "verbs": ["watch"]}]}) + binding = h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", + "metadata": {"name": WATCH_BINDING}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": WATCH_BINDING}, + "subjects": [{"kind": "Group", "name": "system:serviceaccounts:kars-sre", + "apiGroup": "rbac.authorization.k8s.io"}]}) + h.state["watch_binding"] = binding + h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "e2e-watch-only", "namespace": RUNTIME}}) + h.token_identity("watch-only", RUNTIME, "e2e-watch-only") + h.passed("Pre-guard same-name ServiceAccount has no controller-issued authority; its stable token alias contains only synthetic non-JWT data") + h.passed("Unsafe legacy group grants intentionally cover all runtime ServiceAccounts; no token is requested for the same-name fixture") + + +def delete_owned(h, path, obj): + response = h.api("DELETE", path, body={"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": obj["metadata"]["uid"], + "resourceVersion": obj["metadata"]["resourceVersion"]}}, status=(200, 202, 404)) + if response.status_code == 404: + require(response.json().get("kind") == "Status" and response.json().get("reason") == "NotFound", + "Fixture cleanup did not return a Kubernetes NotFound status") + + +@contextmanager +def owned_fixtures(h): + fixtures = [] + primary_failure = False + try: + yield fixtures + except BaseException: + primary_failure = True + raise + finally: + failed = False + for path, obj in reversed(fixtures): + # Deleting the SA after a Secret CAS conflict could let native + # TokenController garbage-collect the foreign replacement. + if failed and obj.get("kind") == "ServiceAccount": + continue + try: + delete_owned(h, path, obj) + except Exception: + failed = True + if failed: + if primary_failure: + print("SRE-DIAG Fenced fixture cleanup failed; retaining the primary acceptance failure", flush=True) + else: + raise AssertionError("Fixture cleanup failed; UID/resourceVersion fences were not retried or weakened") + + +def token_secret_denials(h, before_enrollment=False): + policy = "kars-sre-no-legacy-tokens" + path = f"/api/v1/namespaces/{RUNTIME}/secrets" + prefix = f"e2e-token-{h.phase}-{'pre' if before_enrollment else 'live'}" + account = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(account is not None, "Token denial probes require their actual same-name ServiceAccount fixture") + unsafe = synthetic_token_secret(h, f"{prefix}-create", account) + # Real CREATE/PATCH, not a fixed private name or TokenRequest. A broken + # policy fails immediately; even an accepted fixture cannot mint a JWT. + with owned_fixtures(h) as fixtures: + for user in ("tenant", "registrar"): + response = h.api("POST", path, body=unsafe, user=user) + if response.status_code == 201: + fixtures.append((path + "/" + unsafe["metadata"]["name"], response.json())) + assert_denial(response, f"{user} arbitrary token Secret CREATE", policy) + benign = h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": f"{prefix}-account", "namespace": RUNTIME}, + "automountServiceAccountToken": False}) + fixtures.append((f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/{prefix}-account", benign)) + variants = [ + ("type-and-annotation", "Opaque", {}, {"type": TOKEN_TYPE, + "metadata": {"annotations": {SA_NAME: "sre-api-router"}}}), + ("annotation", TOKEN_TYPE, {}, + {"metadata": {"annotations": {SA_NAME: "sre-api-router"}}}), + ] + if before_enrollment: + # Test a type-only update before enrollment: an Opaque alias is + # itself quarantined by the controller inventory, so it must not + # be introduced into an already-Ready runtime even for this test. + variants.append(("type", "Opaque", {SA_NAME: "sre-api-router"}, {"type": TOKEN_TYPE})) + for suffix, kind, annotations, patch in variants: + body = {"apiVersion": "v1", "kind": "Secret", "type": kind, + "metadata": {"name": f"{prefix}-{suffix}", "namespace": RUNTIME, + "annotations": annotations}} + if kind == TOKEN_TYPE: + body = synthetic_token_secret(h, f"{prefix}-{suffix}", benign) + obj = h.api("POST", path, body=body, user="tenant", status=201).json() + fixtures.append((path + "/" + obj["metadata"]["name"], obj)) + assert_secret_unchanged(h.get("secret", obj["metadata"]["name"], RUNTIME), obj) + response = h.api("PATCH", path + "/" + obj["metadata"]["name"], body={ + **patch, "metadata": {**patch.get("metadata", {}), + "uid": obj["metadata"]["uid"], "resourceVersion": obj["metadata"]["resourceVersion"]}}, + user="tenant") + if "type" in patch: + assert_token_type_immutable(response, f"token Secret {suffix} PATCH") + else: + assert_denial(response, f"token Secret {suffix} PATCH", policy) + assert_secret_unchanged(h.get("secret", obj["metadata"]["name"], RUNTIME), obj) + if before_enrollment: + for patch in ({"type": "Opaque"}, {"metadata": {"annotations": {SA_NAME: "e2e-other"}}}): + before = h.get("secret", TOKEN_ALIAS, RUNTIME) + assert_secret_unchanged(before, h.state["prestaged_alias"]) + response = h.api("PATCH", path + "/" + TOKEN_ALIAS, body={ + **patch, "metadata": {**patch.get("metadata", {}), "uid": before["metadata"]["uid"], + "resourceVersion": before["metadata"]["resourceVersion"]}}, user="tenant") + if "type" in patch: + assert_token_type_immutable(response, "prestaged oldObject type escape") + else: + assert_denial(response, "prestaged oldObject annotation escape", policy) + assert_secret_unchanged(h.get("secret", TOKEN_ALIAS, RUNTIME), before) + h.passed("Tenant and registrar token Secret CREATE and schema-valid annotation updates receive exact admission denials") + h.passed("Token-type mutations receive specific Kubernetes immutable-field errors without changing fixtures") + + +def fixture_review(h): + # Only the known pre-guard fixture is reviewed here. CLI preview must + # refuse while the broad watch grant exists; that refusal is tested too. + spec = copy.deepcopy(h.state["legacy_review_before_guards"]) + for kind, name, namespace, uid in [ + ("namespace", SYSTEM, None, spec["controller"]["namespace"]["uid"]), + ("deployment", "kars-controller", SYSTEM, spec["controller"]["deployment"]["uid"]), + ("karssandbox", "sre", SYSTEM, spec["sandbox"]["uid"]), + ("namespace", RUNTIME, None, spec["runtimeNamespace"]["uid"]), + ]: + obj = h.get(kind, name, namespace) + require(obj and obj["metadata"]["uid"] == uid, "Pre-guard fixture identity changed during staging") + for binding in spec["legacyBindings"]: + obj = h.get(binding["kind"].lower(), binding["name"], binding.get("namespace")) + require(obj and obj["metadata"]["uid"] == binding["uid"] + and obj["roleRef"] == binding["roleRef"] and obj["subjects"] == binding["subjects"], + "Pre-guard fixture binding was replaced or its reviewed authority changed") + binding["resourceVersion"] = obj["metadata"]["resourceVersion"] + consumer = h.get("deployment", "sre", RUNTIME) + require(consumer["metadata"]["uid"] == spec["legacyConsumer"]["uid"], "Pre-guard consumer was replaced") + spec["legacyConsumer"]["resourceVersion"] = consumer["metadata"]["resourceVersion"] + return spec + + +def wait_blocked(h, detail): + def blocked(): + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + status = reg.get("status", {}) if reg else {} + return reg if (status.get("phase") == "Blocked" + and status.get("observedGeneration") == reg["metadata"]["generation"] + and detail in status.get("detail", "")) else False + return h.poll(f"controller rejection for {detail}", blocked, seconds=75) + + +def assert_unissued(h, spec, before): + for binding in spec["legacyBindings"]: + require(h.get(binding["kind"].lower(), binding["name"], binding.get("namespace")) == before[binding["name"]], + "Blocked migration changed a legacy grant") + require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, + "Blocked migration stopped the legacy consumer") + expected = None if h.state["prestaged_account_cleaned"] else h.state["prestaged_account"] + require(h.get("serviceaccount", "sre-api-router", RUNTIME) == expected, + "Controller adopted, mutated or replaced the unowned same-name ServiceAccount before safe enrollment") + require(all(h.get("secret", name, RUNTIME) is None for name in (PRIVATE, AGENT)), + "Private credential appeared while legacy privacy was unsafe") + for name in ("kars-sre-private-reader", "kars-sre-private-author", "kars-sre-private-renew"): + require(h.get("clusterrolebinding", name) is None, "Private grant appeared while legacy privacy was unsafe") + require(h.get("rolebinding", "sre-api-self-renew", RUNTIME) is None, + "Private token-renewal grant appeared while legacy privacy was unsafe") + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + require(not reg.get("status", {}).get("routerServiceAccountUid"), + "Controller registered the unowned same-name ServiceAccount as private authority") + + +def assert_fresh_private_identity(h, reg): + account = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(h.state["prestaged_account_cleaned"] and account + and account["metadata"]["uid"] != h.state["prestaged_account"]["metadata"]["uid"] + and account["metadata"]["uid"] == reg["status"].get("routerServiceAccountUid") + and account["metadata"].get("annotations", {}).get(OWNER) == reg["metadata"]["uid"] + and account.get("automountServiceAccountToken") is False, + "Trusted private identity was not freshly controller-created after unowned fixture cleanup") + require(h.get("secret", TOKEN_ALIAS, RUNTIME) is None, "Prestaged token alias survived private issuance") + + +def block_prestaged_paths(h, spec, before): + h.api("POST", REGISTRATION.rsplit("/", 1)[0], body={ + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, "spec": spec}, user="registrar", status=201) + wait_blocked(h, "Unsafe legacy SRE token Secret alias") + assert_unissued(h, spec, before) + alias = h.get("secret", TOKEN_ALIAS, RUNTIME) + assert_secret_unchanged(alias, h.state["prestaged_alias"]) + delete_owned(h, f"/api/v1/namespaces/{RUNTIME}/secrets/{TOKEN_ALIAS}", alias) + h.poll("prestaged synthetic alias cleanup", lambda: h.get("secret", TOKEN_ALIAS, RUNTIME) is None, seconds=30) + # The broad watch grant still blocks enrollment while the operator removes + # the unowned same-name SA. Never adopt it by name or attach private grants. + account = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(account == h.state["prestaged_account"], "Unowned ServiceAccount changed before operator cleanup") + delete_owned(h, f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router", account) + h.poll("unowned same-name ServiceAccount cleanup", + lambda: h.get("serviceaccount", "sre-api-router", RUNTIME) is None, seconds=30) + h.state["prestaged_account_cleaned"] = True + h.passed("Real controller blocks the unchanged synthetic alias before adopting the same-name SA or issuing private authority; operator UID/RV-fenced cleanup removes both") + + wait_blocked(h, "broad group grant") + assert_unissued(h, spec, before) + binding = h.get("clusterrolebinding", WATCH_BINDING) + require(binding == h.state["watch_binding"], "Blocked watch-only group grant was changed or adopted") + h.passed("Real migration rejects a Secrets-watch-only group grant before retiring grants or issuing private material") + # Keep the registration deliberately incomplete before removing the last + # unsafe fixture, so cleanup cannot auto-start a successful migration. + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + partial = {**spec, "legacyBindings": spec["legacyBindings"][:-1]} + h.api("PATCH", REGISTRATION, body={"metadata": {"uid": reg["metadata"]["uid"], + "resourceVersion": reg["metadata"]["resourceVersion"]}, "spec": partial}, user="registrar", status=200) + delete_owned(h, f"/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/{WATCH_BINDING}", binding) + blocked = wait_blocked(h, "Unreviewed legacy SRE binding") + assert_unissued(h, spec, before) + return blocked + + +def assert_watch_result(response, observed, allowed): + if allowed: + require(response.status_code == 200 and observed, + "Watch-only baseline did not observe the newly created dummy Secret") + else: + # HTTP 200 with no events is still an authorized watch, never a denial. + assert_denial(response, "held old principal Secret WATCH") + require(not observed, "Denied Secret watch nevertheless exposed the dummy Secret") + + +def secret_watch(h, namespace, name, *, user="old-agent", allowed=False, cluster=False): + collection = "/api/v1/secrets" if cluster else f"/api/v1/namespaces/{namespace}/secrets" + version = h.api("GET", collection, status=200).json()["metadata"]["resourceVersion"] + client = h.client(user) + started = threading.Event() + encoded = base64.b64encode(WATCH_MARKER.encode()).decode() + + def observe(): + started.set() + end = min(h.deadline, time.monotonic() + 10) + with client.stream("GET", collection, params={"watch": "true", "resourceVersion": version, + "fieldSelector": f"metadata.name={name}", "timeoutSeconds": 5}, timeout=8) as response: + if response.status_code != 200: + response.read() + return response, False + for index, line in enumerate(response.iter_lines()): + if index > 16 or time.monotonic() > end: + break + if not line: + continue + event = json.loads(line) + obj = event.get("object", {}) + if (event.get("type") == "ADDED" and obj.get("metadata", {}).get("name") == name + and obj["metadata"].get("namespace") == namespace + and obj.get("data", {}).get("dummy") == encoded): + return response, True + return response, False + + with owned_fixtures(h) as fixtures: + with ThreadPoolExecutor(max_workers=1) as pool: + result = pool.submit(observe) + require(started.wait(timeout=3), "Secret watch probe did not start") + obj = h.create({"apiVersion": "v1", "kind": "Secret", + "metadata": {"name": name, "namespace": namespace}, "stringData": {"dummy": WATCH_MARKER}}) + fixtures.append((f"/api/v1/namespaces/{namespace}/secrets/{name}", obj)) + response, observed = result.result(timeout=12) + assert_watch_result(response, observed, allowed) diff --git a/tests/e2e/sre_authority/credential_paths_test.py b/tests/e2e/sre_authority/credential_paths_test.py new file mode 100644 index 000000000..ab50501aa --- /dev/null +++ b/tests/e2e/sre_authority/credential_paths_test.py @@ -0,0 +1,253 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Fixture contract checks, not substitutes for hosted Kubernetes acceptance.""" + +import base64 +import copy +import io +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +from sre_authority.common import OWNER, RUNTIME, require +from sre_authority.credential_paths import ( + SA_NAME, SA_UID, SYNTHETIC_TOKEN, TOKEN_ALIAS, TOKEN_TYPE, + assert_fresh_private_identity, assert_secret_unchanged, assert_unissued, + delete_owned, owned_fixtures, seed_privacy_gaps, synthetic_token_secret, token_secret_denials, +) +from sre_authority.harness_test import Response + +POLICY = "kars-sre-no-legacy-tokens" +CA = "-----BEGIN CERTIFICATE-----\npublic-test-ca\n-----END CERTIFICATE-----\n" +SECRET_PATH = f"/api/v1/namespaces/{RUNTIME}/secrets" + + +class FixtureHarness: + def __init__(self): + self.state = {} + self.phase = "legacy" + self.objects = {} + self.requests = [] + self.identities = [] + self.sequence = 0 + + def create(self, obj): + obj = copy.deepcopy(obj) + self.sequence += 1 + obj["metadata"].update(uid=f"uid-{self.sequence}", resourceVersion=str(self.sequence)) + self.objects[obj["kind"].lower(), obj["metadata"]["name"]] = obj + return copy.deepcopy(obj) + + def get(self, kind, name, _namespace=None): + if kind == "configmap" and name == "kube-root-ca.crt": + return {"data": {"ca.crt": CA}} + if kind == "karssreregistrations.kars.azure.com": + kind = "karssreregistration" + return copy.deepcopy(self.objects.get((kind, name))) + + def poll(self, _label, predicate, **_kwargs): + result = predicate() + require(result, "Unit fixture did not satisfy the real harness predicate") + return result + + def passed(self, _message): + pass + + def token_identity(self, *args): + self.identities.append(args) + + def api(self, method, path, *, body=None, user="admin", status=None): + self.requests.append((method, path, copy.deepcopy(body), user)) + resource = path.split("/")[-2] + kind = {"secrets": "secret", "serviceaccounts": "serviceaccount"}.get(resource) + key = (kind, path.split("/")[-1]) + current = self.objects.get(key) + if method == "POST": + private = (body.get("type") == TOKEN_TYPE + and body["metadata"].get("annotations", {}).get(SA_NAME) == "sre-api-router") + response = Response(403, {"kind": "Status", "reason": "Forbidden", "message": POLICY}) if private \ + else Response(201, self.create(body)) + elif method == "PATCH": + if current is None: + response = Response(404, {"kind": "Status", "reason": "NotFound"}) + elif "type" in body: + response = Response(422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [ + {"field": "type", "reason": "FieldValueInvalid", "message": "field is immutable"}]}}) + else: + response = Response(403, {"kind": "Status", "reason": "Forbidden", "message": POLICY}) + elif method == "DELETE": + if current is None: + response = Response(404, {"kind": "Status", "reason": "NotFound"}) + elif body["preconditions"] != {k: current["metadata"][k] for k in ("uid", "resourceVersion")}: + response = Response(409, {"kind": "Status", "reason": "Conflict"}) + else: + del self.objects[key] + response = Response(200, {}) + else: + raise AssertionError("Unexpected unit fixture request") + if status is not None: + require(response.status_code in (status if isinstance(status, tuple) else (status,)), + f"Unit API HTTP {response.status_code}") + return response + + +class CredentialFixtureTests(unittest.TestCase): + def seeded(self): + h = FixtureHarness() + seed_privacy_gaps(h) + return h + + def test_seed_uses_live_unowned_account_and_complete_noncredential_data(self): + h = self.seeded() + account = h.state["prestaged_account"] + secret = h.state["prestaged_alias"] + self.assertFalse(account["automountServiceAccountToken"]) + for field in ("ownerReferences", "annotations", "labels", "finalizers"): + self.assertNotIn(field, account["metadata"]) + self.assertNotIn("secrets", account) + self.assertEqual(secret["metadata"]["annotations"], { + SA_NAME: "sre-api-router", SA_UID: account["metadata"]["uid"]}) + self.assertEqual({key: base64.b64decode(value).decode() for key, value in secret["data"].items()}, + {"token": SYNTHETIC_TOKEN, "ca.crt": CA, "namespace": RUNTIME}) + self.assertNotIn(".", SYNTHETIC_TOKEN) + self.assertNotIn("finalizers", secret["metadata"]) + self.assertEqual(h.identities, [("watch-only", RUNTIME, "e2e-watch-only")]) + + def test_synthetic_secret_requires_current_account_uid_namespace_and_ca(self): + h = self.seeded() + account = h.state["prestaged_account"] + for changed in ({"uid": ""}, {"namespace": "other"}, {"deletionTimestamp": "now"}): + bad = copy.deepcopy(account) + bad["metadata"].update(changed) + with self.subTest(changed=changed), self.assertRaises(AssertionError): + synthetic_token_secret(h, "probe", bad) + with patch.object(h, "get", return_value={"data": {}}), self.assertRaises(AssertionError): + synthetic_token_secret(h, "probe", account) + + def test_missing_replaced_modified_or_populated_fixture_never_proves_denial(self): + secret = self.seeded().state["prestaged_alias"] + assert_secret_unchanged(copy.deepcopy(secret), secret) + variants = [None] + for key in ("uid", "resourceVersion", "annotations", "finalizers", "ownerReferences"): + bad = copy.deepcopy(secret) + bad["metadata"][key] = "changed" + variants.append(bad) + for changed in ({"data": {}}, {"data": {"token": "unexpected"}}, {"type": "Opaque"}): + variants.append({**secret, **changed}) + for bad in variants: + with self.subTest(case=variants.index(bad)), self.assertRaises(AssertionError): + assert_secret_unchanged(bad, secret) + + def test_real_probe_contract_preserves_old_object_and_uses_live_benign_sa(self): + for before in (True, False): + with self.subTest(before=before): + h = self.seeded() + token_secret_denials(h, before_enrollment=before) + posts = [body for method, path, body, _ in h.requests + if method == "POST" and path == SECRET_PATH] + tokens = [obj for obj in posts if obj["type"] == TOKEN_TYPE] + self.assertEqual(len(tokens), 3) # Two CREATE denials and the annotation fixture. + for obj in tokens: + self.assertTrue(obj["metadata"]["annotations"][SA_UID]) + self.assertEqual(base64.b64decode(obj["data"]["token"]).decode(), SYNTHETIC_TOKEN) + annotation = next(obj for obj in tokens if obj["metadata"]["name"].endswith("-annotation")) + self.assertNotEqual(annotation["metadata"]["annotations"][SA_NAME], "sre-api-router") + escapes = [body for method, path, body, _ in h.requests + if method == "PATCH" and path.endswith("/" + TOKEN_ALIAS)] + self.assertEqual(len(escapes), 2 if before else 0) + for body in escapes: + self.assertEqual(body["metadata"]["uid"], h.state["prestaged_alias"]["metadata"]["uid"]) + self.assertEqual(body["metadata"]["resourceVersion"], + h.state["prestaged_alias"]["metadata"]["resourceVersion"]) + self.assertEqual(h.get("secret", TOKEN_ALIAS), h.state["prestaged_alias"]) + self.assertFalse(any(name.startswith("e2e-token-") for _, name in h.objects)) + + def test_cleanup_accepts_only_real_absence_and_never_drops_cas_fences(self): + h = self.seeded() + obj = h.state["prestaged_alias"] + path = SECRET_PATH + "/" + TOKEN_ALIAS + delete_owned(h, path, obj) + delete_owned(h, path, obj) # Already absent is cleanup, never admission proof. + self.assertEqual(h.requests[-1][2]["preconditions"], { + key: obj["metadata"][key] for key in ("uid", "resourceVersion")}) + replacement = h.create({**obj, "metadata": {"name": TOKEN_ALIAS, "namespace": RUNTIME}}) + with self.assertRaisesRegex(AssertionError, "409"): + delete_owned(h, path, obj) + self.assertEqual(h.get("secret", TOKEN_ALIAS), replacement) + with patch.object(h, "api", return_value=Response(404, {})), self.assertRaises(AssertionError): + delete_owned(h, path, obj) + + def test_cleanup_conflict_preserves_primary_error_and_attempts_independent_fixtures(self): + h = self.seeded() + output = io.StringIO() + with patch("sre_authority.credential_paths.delete_owned", side_effect=AssertionError("private-body")) as cleanup: + with redirect_stdout(output), self.assertRaisesRegex(AssertionError, "^primary denial failure$"): + with owned_fixtures(h) as fixtures: + fixtures.extend([("first", {}), ("second", {})]) + raise AssertionError("primary denial failure") + self.assertEqual([call.args[1] for call in cleanup.call_args_list], ["second", "first"]) + self.assertNotIn("private-body", output.getvalue()) + with self.assertRaisesRegex(AssertionError, "Fixture cleanup failed"): + with owned_fixtures(h) as fixtures: + fixtures.append(("first", {})) + + def test_secret_conflict_preserves_account_to_prevent_gc_of_foreign_replacement(self): + h = self.seeded() + original = h.state["prestaged_alias"] + replacement = h.create(original) + account = h.state["prestaged_account"] + with self.assertRaisesRegex(AssertionError, "Fixture cleanup failed"): + with owned_fixtures(h) as fixtures: + fixtures.append((f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router", account)) + fixtures.append((SECRET_PATH + "/" + TOKEN_ALIAS, original)) + self.assertEqual(h.get("secret", TOKEN_ALIAS), replacement) + self.assertEqual(h.get("serviceaccount", "sre-api-router"), account) + self.assertEqual(len(h.requests), 1) + + def test_disappeared_fixture_patch_404_is_fatal_even_when_cleanup_is_404(self): + h = self.seeded() + original = h.api + def missing(method, path, **kwargs): + if method == "PATCH": + h.objects.pop(("secret", path.split("/")[-1]), None) + return Response(404, {"kind": "Status", "reason": "NotFound"}) + return original(method, path, **kwargs) + with patch.object(h, "api", side_effect=missing), self.assertRaisesRegex(AssertionError, "got HTTP 404"): + token_secret_denials(h, before_enrollment=True) + + def test_blocked_controller_cannot_adopt_or_record_same_name_fixture(self): + h = self.seeded() + h.create({"kind": "Deployment", "metadata": {"name": "sre"}, "spec": {"replicas": 1}}) + reg = h.create({"kind": "KarsSRERegistration", "metadata": {"name": "canonical"}, "status": {}}) + assert_unissued(h, {"legacyBindings": []}, {}) + h.objects["karssreregistration", "canonical"]["status"]["routerServiceAccountUid"] = \ + h.state["prestaged_account"]["metadata"]["uid"] + with self.assertRaises(AssertionError): + assert_unissued(h, {"legacyBindings": []}, {}) + h.objects["karssreregistration", "canonical"] = reg + h.objects["serviceaccount", "sre-api-router"]["metadata"]["annotations"] = {OWNER: reg["metadata"]["uid"]} + with self.assertRaises(AssertionError): + assert_unissued(h, {"legacyBindings": []}, {}) + + def test_trusted_private_identity_requires_cleanup_new_uid_and_registration_owner(self): + h = self.seeded() + reg = {"metadata": {"uid": "registration"}, "status": {}} + with self.assertRaises(AssertionError): + assert_fresh_private_identity(h, reg) + h.state["prestaged_account_cleaned"] = True + del h.objects["secret", TOKEN_ALIAS] + original = h.state["prestaged_account"] + fresh = h.create({"kind": "ServiceAccount", "metadata": { + "name": "sre-api-router", "namespace": RUNTIME, "annotations": {OWNER: "registration"}}, + "automountServiceAccountToken": False}) + reg["status"]["routerServiceAccountUid"] = fresh["metadata"]["uid"] + assert_fresh_private_identity(h, reg) + for uid in (original["metadata"]["uid"], "unregistered"): + h.objects["serviceaccount", "sre-api-router"]["metadata"]["uid"] = uid + with self.assertRaises(AssertionError): + assert_fresh_private_identity(h, reg) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py new file mode 100644 index 000000000..0343c467d --- /dev/null +++ b/tests/e2e/sre_authority/fixtures.py @@ -0,0 +1,263 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import io +import json +import tarfile +from copy import deepcopy + +from .common import ( + AGENT, CLAIM_VERSION, CONTEXT, FIELD_MANAGER, NAMESPACE_UID, OPERATORS, + PRIVATE, RUNTIME, SOURCE_NAME, SOURCE_NS, SOURCE_UID, STANDIN, SYSTEM, TENANT, + enrollment_json, fingerprint, require, +) +from .credential_paths import seed_privacy_gaps +from .legacy_crds import preflight_legacy_crds +from .registration_schema import create_registration_crd + +LEGACY_COMMIT = "8b206065608593667a40665b3f48225ef9ce278d" +CONTROL = "e2e-control-rotation" +CONTROL_NS = f"kars-{CONTROL}" +GROUP_BINDING = "e2e-sre-legacy-group" + + +def namespace_claim(h, source, name): + namespace = h.get("namespace", name) + if not namespace: + namespace = h.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}}) + uid = namespace["metadata"]["uid"] + h.api("PATCH", f"/api/v1/namespaces/{name}", body={ + "metadata": {"uid": uid, "resourceVersion": namespace["metadata"]["resourceVersion"], + "annotations": {CLAIM_VERSION: "v1", SOURCE_NS: SYSTEM, + SOURCE_NAME: source["metadata"]["name"], SOURCE_UID: source["metadata"]["uid"]}}, + }, status=200) + h.k("patch", "karssandbox", source["metadata"]["name"], "-n", SYSTEM, "--type=merge", "-p", + json.dumps({"metadata": {"uid": source["metadata"]["uid"], + "annotations": {NAMESPACE_UID: uid}}})) + return uid + + +def service_account(h, name, namespace): + return h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": name, "namespace": namespace}}) + + +def role_binding(h, name, role, account, namespace=None, account_namespace=OPERATORS): + return h.create({"apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding" if namespace else "ClusterRoleBinding", + "metadata": {"name": name, **({"namespace": namespace} if namespace else {})}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": role}, + "subjects": [{"kind": "ServiceAccount", "name": account, "namespace": account_namespace}]}) + + +def install_historical_chart(h): + h.run(["git", "fetch", "--no-tags", "--depth=1", "https://github.com/Azure/kars.git", LEGACY_COMMIT], timeout=90) + # git archive is binary; invoke separately without decoding tar as text. + import subprocess + data = subprocess.run(["git", "archive", LEGACY_COMMIT, "deploy/helm/kars"], + cwd=h.root, capture_output=True, timeout=30, check=True).stdout + destination = h.work / "legacy-chart" + require(not destination.exists(), "Historical fixture extraction destination already exists") + destination.mkdir() + with tarfile.open(fileobj=io.BytesIO(data)) as chart: + for member in chart.getmembers(): + require((member.name.startswith("deploy/helm/kars/") + or member.name.rstrip("/") in ("deploy", "deploy/helm", "deploy/helm/kars")) + and ".." not in member.name.split("/") + and (member.isdir() or member.isfile()), "Unsafe historical chart archive") + chart.extractall(destination, filter="data") + sources = { + member.name.rsplit("/", 1)[-1]: chart.extractfile(member).read().decode() + for member in chart.getmembers() + if member.isfile() and member.name.startswith("deploy/helm/kars/templates/crd") + } + preflight_legacy_crds(h, destination / "deploy/helm/kars", sources) + version = h.run(["helm", "version", "--template", "{{.Version}}"]).strip() + wait_arg = h.run(["bash", "-c", 'source "$1"; sre_migration_helm_wait_arg "$2"', + "legacy-helm-wait", str(h.root / "tests/e2e/sre-authority.sh"), version]).strip() + # Helm's legacy waiter includes templated CRDs (Established), and runs + # before post-install hooks. Let Helm retain its native SSA ownership. + h.run(["helm", "install", "kars", str(destination / "deploy/helm/kars"), + "--kube-context", CONTEXT, "--namespace", SYSTEM, "--create-namespace", + wait_arg, "--timeout", "120s", + "--set", "controller.replicas=0", "--set", "controller.image.repository=kars-controller", + "--set", "controller.image.tag=e2e", "--set", "controller.image.pullPolicy=Never", + "--set", "inferenceRouter.image.repository=kars-inference-router", + "--set", "inferenceRouter.image.tag=e2e", "--set", "sandbox.image.repository=kars-sandbox-e2e", + "--set", "sandbox.image.tag=dev", "--set-string", f"runtimes.hermes.image={STANDIN}", + "--set", "sre.enabled=false", "--set-string", "inferenceRouter.azure.openai.endpoint=https://e2e-fake.invalid/", + "--set-string", "foundry.endpoint=https://e2e-fake.invalid/"], timeout=180) + require(h.get("toolpolicy", "kars-default", SYSTEM) is not None, + "Historical Helm post-install ToolPolicy hook did not create its policy") + h.passed("Historical Helm initial install and unchanged ToolPolicy post-install hook completed") + h.k("wait", "--for=condition=Established", "crd/karssandboxes.kars.azure.com", "--timeout=60s", timeout=70) + h.run(["helm", "upgrade", "kars", str(destination / "deploy/helm/kars"), "--kube-context", CONTEXT, + "--namespace", SYSTEM, "--reuse-values", "--set", "sre.enabled=true"], timeout=120) + + +def prepare_legacy(h): + require(not h.get("validatingadmissionpolicy", "kars-sre-source-authority"), + "Legacy fixtures must precede the new SRE admission guards") + require(not h.get("namespace", RUNTIME), "Disposable cluster contains a pre-existing SRE namespace") + install_historical_chart(h) + source = h.get("karssandbox", "sre", SYSTEM) + require(source is not None, "Historical chart did not create its source CR") + h.state["legacy_source_uid"] = source["metadata"]["uid"] + h.state["legacy_namespace_uid"] = namespace_claim(h, source, RUNTIME) + h.state["system_uid"] = h.get("namespace", SYSTEM)["metadata"]["uid"] + h.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": OPERATORS}}) + h.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": TENANT}}) + service_account(h, "sandbox", RUNTIME) + for account in ("registrar", "unrelated"): + service_account(h, account, OPERATORS) + for account in ("tenant", "normal"): + service_account(h, account, TENANT) + h.token_identity("old-agent", RUNTIME, "sandbox") + h.token_identity("unrelated", OPERATORS, "unrelated") + binding = h.get("clusterrolebinding", "kars-sre-reader") + require(binding and binding.get("subjects"), "Historical reader grant is absent") + unrelated = {"kind": "ServiceAccount", "name": "unrelated", "namespace": OPERATORS} + h.k("patch", "clusterrolebinding", "kars-sre-reader", "--type=merge", "-p", json.dumps({ + "metadata": {"uid": binding["metadata"]["uid"], "resourceVersion": binding["metadata"]["resourceVersion"]}, + "subjects": binding["subjects"] + [unrelated], + })) + h.state["unrelated_subject"] = unrelated + h.create({"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": "sre", "namespace": RUNTIME, "labels": { + "kars.azure.com/component": "sandbox", "kars.azure.com/sandbox": "sre", + "kars.azure.com/parent-namespace": SYSTEM}}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"kars.azure.com/sandbox": "sre"}}, + "template": {"metadata": {"labels": {"kars.azure.com/sandbox": "sre"}}, + "spec": {"serviceAccountName": "sandbox", + "securityContext": {"runAsUser": 1000, "runAsNonRoot": True, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "agent", "image": STANDIN, "imagePullPolicy": "IfNotPresent", + "command": ["/bin/sh", "-c", "echo legacy-consumer-fixture; sleep infinity"]}]}}}}) + h.k("rollout", "status", "deployment/sre", "-n", RUNTIME, "--timeout=120s", timeout=130) + seed_control_consumer(h) + h.state["legacy_review_before_guards"] = enrollment_json(h.cli("authority", "preview")) + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", + "metadata": {"name": GROUP_BINDING}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "kars-sre-reader"}, + "subjects": [{"kind": "Group", "name": "system:serviceaccounts:kars-sre", "apiGroup": "rbac.authorization.k8s.io"}]}) + seed_privacy_gaps(h) + require(not h.get("validatingadmissionpolicy", "kars-sre-source-authority"), + "Legacy grants were not seeded before admission") + # Bootstrap only the new cluster API, not any of its admission policies. + # This makes kubectl's real discovery usable by authority stage's can-i. + rendered = h.run(["helm", "template", "kars", str(h.root / "deploy/helm/kars"), + "--namespace", SYSTEM, "--show-only", "templates/crd-karssreregistration.yaml"]) + converter = "const y=require('node:module').createRequire(process.cwd()+'/cli/package.json')('yaml'),f=require('fs');console.log(JSON.stringify(y.parse(f.readFileSync(0,'utf8'))));" + obj = json.loads(h.run(["node", "-e", converter], data=rendered, timeout=20)) + obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" + obj["metadata"].setdefault("annotations", {}).update({ + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM}) + create_registration_crd(h, obj) + h.k("wait", "--for=condition=Established", "crd/karssreregistrations.kars.azure.com", "--timeout=60s", timeout=70) + before = h.get("clusterrolebinding", "kars-sre-reader") + action_before = h.get("crd", "karssreactions.kars.azure.com") + action_spec = deepcopy(action_before["spec"]) + action_params = action_spec["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]["action"]["properties"]["params"] + require(action_params.pop("additionalProperties", None) is True, + "Historical native action CRD does not exercise the boolean params prerequisite") + action_params["x-kubernetes-preserve-unknown-fields"] = True + h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", "--dry-run", timeout=150) + after = h.get("clusterrolebinding", "kars-sre-reader") + require(before["metadata"]["uid"] == after["metadata"]["uid"] and before["subjects"] == after["subjects"], + "Authority stage preview changed legacy grants") + h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", timeout=180) + action_after = h.get("crd", "karssreactions.kars.azure.com") + require(action_after["metadata"]["uid"] == action_before["metadata"]["uid"] + and action_after["spec"] == action_spec, + "Native action API staging replaced its identity or changed fields outside the params repair") + h.passed("Actual CLI stages the historical action CRD params repair in place before dependent policies") + h.save() + h.passed("Legacy source/grants/consumer seeded using real UIDs before new policies; immutable old chart only, no old binary execution") + h.passed("Actual authority stage preview/apply retains legacy grants without private issuance") + + +def seed_control_consumer(h): + source = h.create({"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": CONTROL, "namespace": SYSTEM}, + "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1", + "command": ["/bin/sh"], "args": ["-c", "sleep infinity"]}}, + "inferenceRef": {"name": "sre-inference"}, + "sandbox": {"isolation": "standard"}}}) + namespace_uid = namespace_claim(h, source, CONTROL_NS) + secret = h.create({"apiVersion": "v1", "kind": "Secret", + "metadata": {"name": "router-services-admin", "namespace": CONTROL_NS, + "labels": {"app.kubernetes.io/managed-by": "kars-controller"}, + "annotations": {SOURCE_UID: source["metadata"]["uid"], NAMESPACE_UID: namespace_uid}}, + "stringData": {"control-token": "legacy-exposed-control-fixture-" + "x" * 40}}) + h.state["control_digest"] = fingerprint(secret) + h.state["control_secret_uid"] = secret["metadata"]["uid"] + h.state["control_source_uid"] = source["metadata"]["uid"] + h.create({"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": CONTROL, "namespace": CONTROL_NS, "labels": { + "kars.azure.com/component": "sandbox", "kars.azure.com/sandbox": CONTROL, + "kars.azure.com/parent-namespace": SYSTEM}}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"kars.azure.com/sandbox": CONTROL}}, + "template": {"metadata": {"labels": {"kars.azure.com/sandbox": CONTROL}}, + "spec": {"securityContext": {"runAsUser": 1000, "runAsNonRoot": True, "fsGroup": 1000, + "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "credential-consumer", "image": STANDIN, + "command": ["/bin/sh", "-c", "sha256sum /credential/control-token > /proof/cached.sha; echo control-consumer-fixture; sleep infinity"], + "volumeMounts": [{"name": "old-control", "mountPath": "/credential", "readOnly": True}, + {"name": "proof", "mountPath": "/proof"}]}], + "volumes": [{"name": "old-control", "secret": {"secretName": "router-services-admin"}}, + {"name": "proof", "emptyDir": {}}]}}}}, manager="sre-e2e-fixture") + # Proven controller ownership of spec, without taking ownership of the + # fixture's extra cache-observation sidecar or its volumes. + h.k("apply", "--server-side", f"--field-manager={FIELD_MANAGER}", "-f", "-", data=json.dumps({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": CONTROL, "namespace": CONTROL_NS}, + "spec": {"replicas": 1}})) + h.k("rollout", "status", f"deployment/{CONTROL}", "-n", CONTROL_NS, "--timeout=120s", timeout=130) + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="old-agent", status=200) + h.passed("Held legacy agent principal can genuinely read an owned control credential before migration") + + +def delegate_operators(h): + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": "e2e-sre-review-reader"}, "rules": [ + {"apiGroups": [""], "resources": ["namespaces"], "verbs": ["get", "list"]}, + {"apiGroups": ["apps"], "resources": ["deployments"], "verbs": ["get", "list"]}, + {"apiGroups": ["kars.azure.com"], "resources": ["karssandboxes"], "verbs": ["get", "list"]}, + {"apiGroups": ["apiextensions.k8s.io"], "resources": ["customresourcedefinitions"], "verbs": ["get"]}, + {"apiGroups": ["rbac.authorization.k8s.io"], "resources": ["clusterrolebindings", "rolebindings", "clusterroles", "roles"], "verbs": ["get", "list"]}, + ]}) + role_binding(h, "e2e-sre-review-reader", "e2e-sre-review-reader", "registrar") + role_binding(h, "e2e-sre-registrar", "kars-sre-registrar", "registrar") + h.token_identity("registrar", OPERATORS, "registrar") + h.token_identity("tenant", TENANT, "tenant") + h.token_identity("normal", TENANT, "normal") + for namespace in (RUNTIME, TENANT): + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", + "metadata": {"name": "e2e-sre-admission-probe", "namespace": namespace}, "rules": [ + {"apiGroups": [""], "resources": ["pods", "pods/ephemeralcontainers", "pods/exec", "pods/attach", "pods/portforward", "pods/proxy", "serviceaccounts", "serviceaccounts/token", "secrets"], "verbs": ["create", "patch", "update"]}, + {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list"]}, + {"apiGroups": [""], "resources": ["pods/log"], "verbs": ["get"]}, + {"apiGroups": [""], "resources": ["replicationcontrollers"], "verbs": ["create", "get", "update"]}, + {"apiGroups": [""], "resources": ["pods/exec", "pods/attach", "pods/portforward", "pods/proxy"], "verbs": ["get"]}, + {"apiGroups": ["apps"], "resources": ["deployments", "replicasets", "statefulsets", "daemonsets"], "verbs": ["create", "patch", "update"]}, + {"apiGroups": ["batch"], "resources": ["jobs", "cronjobs"], "verbs": ["create"]}, + {"apiGroups": ["kars.azure.com"], "resources": ["karssandboxes", "karssreactions"], "verbs": ["create", "patch", "update"]}, + ]}) + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": "e2e-sre-admission-probe", "namespace": namespace}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "e2e-sre-admission-probe"}, + "subjects": [{"kind": "ServiceAccount", "name": "tenant", "namespace": TENANT}]}) + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": "e2e-sre-registration-probe"}, "rules": [ + {"apiGroups": ["kars.azure.com"], "resources": ["karssreregistrations"], "verbs": ["create", "patch", "update"]}]}) + role_binding(h, "e2e-sre-registration-probe", "e2e-sre-registration-probe", "tenant", account_namespace=TENANT) + # Port-forward/exec are normal deployment rights, separate from registrar use. + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": "e2e-sre-registrar-runtime", "namespace": RUNTIME}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "e2e-sre-admission-probe"}, + "subjects": [{"kind": "ServiceAccount", "name": "registrar", "namespace": OPERATORS}]}) + require(h.k("auth", "can-i", "use", "karssreregistrations.kars.azure.com/canonical", + user="registrar").strip() == "yes", "Delegated registrar cannot use enrollment") + result = h.k("auth", "can-i", "create", "clusterrolebindings", user="registrar", expected=None) + require(result.returncode == 1 and result.stdout.strip() == "no", "Registrar fixture accidentally has cluster-admin-equivalent RBAC") + h.passed("Explicit registrar delegation works without granting cluster-admin to the registrar or any agent") diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py new file mode 100644 index 000000000..a45c66113 --- /dev/null +++ b/tests/e2e/sre_authority/harness_test.py @@ -0,0 +1,515 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure harness checks. These are not the hosted Kubernetes acceptance result.""" + +import copy +import json +from pathlib import Path +import re +import subprocess +import tempfile +import time +import types +import unittest +from unittest.mock import patch + +from sre_authority.common import ( + CLAIM_VERSION, NAMESPACE_UID, RUNTIME, SOURCE_NAME, SOURCE_NS, SOURCE_UID, SYSTEM, + Harness, assert_claim, assert_denial, authority_failure_site, enrollment_json, printed_object, review_args, +) +from sre_authority.fixtures import seed_control_consumer +from sre_authority.admission import reserved_source_probe +from sre_authority.proxy import MARKER, assert_filtered, log_reader_facts, secret_list_wire_facts +from sre_authority.credential_paths import assert_token_type_immutable, assert_watch_result + + +class Response: + def __init__(self, code, value): + self.status_code = code + self.value = value + + def json(self): + return self.value + + +class HarnessTests(unittest.TestCase): + def test_hermes_runtime_assertion_verifies_exact_pin_and_runtime_without_blanket_standin_acceptance(self): + helper = Path(__file__).resolve().parents[1] / "sre-authority.sh" + cases = [ + ("kars-sandbox-e2e:dev", "kars-sandbox-e2e:dev", "Hermes", 0), + ("wrong:latest", "kars-sandbox-e2e:dev", "Hermes", 1), + ("kars-sandbox-e2e:dev", "", "Hermes", 1), + ("custom/runtime:latest", "custom/runtime:latest", "Hermes", 0), + ("kars-runtime-hermes:latest", "", "Hermes", 0), + ("kars-runtime-hermes:latest", "", "BYO", 1), + ("", "", "Hermes", 1), + ] + for image, configured, runtime, expected in cases: + result = subprocess.run( + ["bash", "-c", 'source "$1"; sre_hermes_image_matches "$2" "$3" "$4"', + "hermes-image-test", str(helper), image, configured, runtime], + capture_output=True, text=True, timeout=5, check=False) + with self.subTest(image=image, configured=configured, runtime=runtime): + self.assertEqual(result.returncode, expected) + + def test_log_reader_diagnostics_never_echo_log_text_or_error_body(self): + root = Path(__file__).resolve().parents[3] + facts = log_reader_facts(root, {"error": "406 Not Acceptable", + "body": json.dumps({"kind": "Status", "reason": "SREProxyDenied", + "message": "Kubernetes rejected the diagnostic/proposal request", "data": MARKER})}) + self.assertEqual(facts["httpStatus"], 406) + self.assertTrue(facts["hasError"]) + self.assertIn("responseSite", facts) + self.assertNotIn(MARKER, json.dumps(facts)) + facts = log_reader_facts(root, {"logs": "sre-standin-alive " + MARKER}) + self.assertTrue(facts["standinMarker"]) + self.assertNotIn(MARKER, json.dumps(facts)) + + def test_native_secret_list_wire_facts_publish_no_synthetic_secret_fields(self): + value = {"kind": "SecretList", "items": [{"metadata": {"uid": "fixture", "annotations": {"copy": MARKER}}, + "data": {"token": MARKER}}]} + facts = secret_list_wire_facts(value, "fixture") + self.assertEqual(facts, {"envelopeKind": "SecretList", "itemHasKind": False, + "itemHasApiVersion": False, "itemMetadataObject": True}) + self.assertNotIn(MARKER, json.dumps(facts)) + with self.assertRaises(AssertionError): + secret_list_wire_facts(value, "other") + with self.assertRaises(AssertionError): + secret_list_wire_facts({"kind": "List", "items": []}, "fixture") + + def test_firewall_diagnostic_requires_owned_node_before_readonly_namespace_entry(self): + h = Harness.__new__(Harness) + calls = [] + h.run = lambda args, **_kwargs: calls.append(args) or "foreign-cluster" + pod = {"metadata": {"uid": "pod-uid"}, "spec": {"nodeName": "kars-e2e-worker"}} + with self.assertRaises(AssertionError): + h.firewall_diagnostics(pod) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][:3], ["docker", "inspect", "kars-e2e-worker"]) + pod["spec"]["hostNetwork"] = True + with self.assertRaises(AssertionError): + h.firewall_diagnostics(pod) + self.assertEqual(len(calls), 1) + + def test_connectivity_diagnostic_is_uid_fenced_nonprivileged_and_has_no_credentials(self): + h = Harness.__new__(Harness) + pod = {"metadata": {"name": "sre-probe", "uid": "pod-uid", "resourceVersion": "17"}, + "spec": {"automountServiceAccountToken": False}, + "status": {"ephemeralContainerStatuses": [{"name": "sre-e2e-network-diagnostic", + "state": {"terminated": {"exitCode": 0}}}]}} + def get(kind, _name, _namespace): + if kind == "service": + return {"spec": {"clusterIP": "10.96.0.1", "ports": [{"name": "https", "port": 443}]}} + if kind == "endpoints": + return {"subsets": [{"addresses": [{"ip": "172.18.0.2"}], + "ports": [{"name": "https", "port": 6443}]}]} + return pod + writes = [] + h.get = get + h.api = lambda method, path, **kwargs: writes.append((method, path, kwargs)) + h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) + h.k = lambda *_args: '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}' + h.control_connectivity_diagnostics = lambda *_args: {"available": False} + h.policy_connectivity_diagnostics = lambda *_args, **_kwargs: {"available": False} + self.assertEqual(h.connectivity_diagnostics(pod), { + "available": True, "uidMatches1001": True, "serviceExit": 0, "endpointExit": 0, + "sameNodeControl": {"available": False}, + "guardControls": {variant: {"available": False} for variant in ("full", "filter-only", "legacy-full")}, + "policyControl": {"available": False}, "denyAllPolicyControl": {"available": False}, + "apiEndpointPolicyControl": {"available": False}}) + self.assertEqual(len(writes), 1) + method, path, args = writes[0] + self.assertEqual(method, "PATCH") + self.assertTrue(path.endswith("/ephemeralcontainers")) + body = args["body"] + self.assertEqual(body["metadata"], {"uid": "pod-uid", "resourceVersion": "17"}) + probe = body["spec"]["ephemeralContainers"][0] + self.assertEqual(probe["securityContext"]["runAsUser"], 1001) + self.assertFalse(probe["securityContext"]["allowPrivilegeEscalation"]) + self.assertEqual(probe["securityContext"]["capabilities"], {"drop": ["ALL"]}) + for field in ("targetContainerName", "volumeMounts", "env", "envFrom"): + self.assertNotIn(field, probe) + for key, value in (("automountServiceAccountToken", True), ("shareProcessNamespace", True)): + old = copy.deepcopy(pod["spec"]) + pod["spec"][key] = value + with self.assertRaises(AssertionError): + h.connectivity_diagnostics(pod) + pod["spec"] = old + self.assertEqual(len(writes), 1) + + def test_connectivity_result_rejects_arbitrary_output_or_untyped_fields(self): + for value in ({"available": True}, {"available": 1}, + {"available": False, "data": MARKER}, + {"available": True, "uidMatches1001": True, "serviceExit": "0", "endpointExit": 0}): + with self.assertRaises(AssertionError): + Harness.parse_connectivity_result(json.dumps(value)) + self.assertEqual(Harness.parse_connectivity_result('{"available":false}'), {"available": False}) + + def test_same_node_control_creates_only_unprivileged_owned_pod_and_uid_fenced_cleanup(self): + h = Harness.__new__(Harness) + pod = {"spec": {"nodeName": "sandbox-worker", "securityContext": {"runAsNonRoot": True}}} + container = {"name": "network", "image": "kars-sandbox-e2e:dev", + "securityContext": {"runAsUser": 1001, "capabilities": {"drop": ["ALL"]}}} + seen, deleted = [], [] + current = {"metadata": {"uid": "control-uid", "resourceVersion": "7"}, "status": {"phase": "Succeeded"}} + h.create = lambda obj: seen.append(obj) or current + h.get = lambda *_args: current + h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) + h.k = lambda *_args: '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}' + h.api = lambda method, path, **kwargs: deleted.append((method, path, kwargs)) + h.control_connectivity_diagnostics(pod, container) + self.assertEqual(len(seen), 1) + self.assertEqual(seen[0]["spec"]["nodeName"], "sandbox-worker") + self.assertFalse(seen[0]["spec"]["automountServiceAccountToken"]) + self.assertNotIn("shareProcessNamespace", seen[0]["spec"]) + self.assertNotIn("volumes", seen[0]["spec"]) + self.assertEqual(deleted[0][0], "DELETE") + self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "control-uid", "resourceVersion": "7"}) + + def test_endpoint_candidate_keeps_full_guard_and_tests_separate_agent_uid(self): + h = Harness.__new__(Harness) + h.root = Path(__file__).resolve().parents[3] + pod = {"spec": {"nodeName": "sandbox-worker"}} + container = {"name": "network", "image": "kars-sandbox-e2e:dev", + "command": ["sh", "-c", "uid test = 1001 uidMatches1001"], + "securityContext": {"runAsUser": 1001, "capabilities": {"drop": ["ALL"]}}} + seen = [] + current = {"metadata": {"uid": "control", "resourceVersion": "1"}, "status": {"phase": "Succeeded"}} + h.create = lambda obj: seen.append(obj) or current + h.get = lambda *_args: current + h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) + h.k = lambda *args: ( + '{"available":true,"uidMatches1000":true,"serviceExit":1,"endpointExit":124}' + if "agent-network" in args else '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}') + h.api = lambda *_args, **_kwargs: None + with patch("sre_authority.network_diagnostics.guard_variant", return_value={"name": "guard"}) as guard: + result = h.control_connectivity_diagnostics(pod, container, policy=True, api_endpoint=True) + guard.assert_called_once_with(h.root, pod, "full") + actual = seen[0]["spec"]["containers"] + self.assertEqual([obj["securityContext"]["runAsUser"] for obj in actual], [1001, 1000]) + self.assertIn("= 1000", actual[1]["command"][2]) + self.assertIn("uidMatches1000", actual[1]["command"][2]) + self.assertTrue(actual[0]["command"][2].startswith("sleep 10\n")) + self.assertTrue(result["agentUid1000"]["uidMatches1000"]) + self.assertNotEqual(result["agentUid1000"]["endpointExit"], 0) + for obj in actual: + self.assertNotIn("volumeMounts", obj) + self.assertNotIn("envFrom", obj) + + def test_policy_comparison_targets_only_owned_control_and_never_changes_source_policy(self): + h = Harness.__new__(Harness) + source = {"spec": {"podSelector": {}, "policyTypes": ["Egress"], + "egress": [{"to": [{"ipBlock": {"cidr": "10.96.0.1/32"}}], + "ports": [{"port": 443, "protocol": "TCP"}]}]}} + before = copy.deepcopy(source) + seen, deleted = [], [] + own = {"metadata": {"uid": "policy-uid", "resourceVersion": "3"}} + h.get = lambda kind, name, namespace: source if name == "sandbox-policy" else own + h.create = lambda obj: seen.append(obj) or own + h.control_connectivity_diagnostics = lambda *_args, **kwargs: {"policy": kwargs["policy"]} + h.api = lambda method, path, **kwargs: deleted.append((method, path, kwargs)) + self.assertEqual(h.policy_connectivity_diagnostics({}, {}), {"policy": True}) + self.assertEqual(source, before) + h.get = lambda kind, name, namespace: ( + {"subsets": [{"addresses": [{"ip": "172.18.0.2"}], "ports": [{"name": "https", "port": 6443}]}]} + if kind == "endpoints" else source if name == "sandbox-policy" else own) + h.policy_connectivity_diagnostics({}, {}, api_endpoint=True) + self.assertEqual(seen[1]["spec"]["egress"], source["spec"]["egress"] + [ + {"to": [{"ipBlock": {"cidr": "172.18.0.2/32"}}], "ports": [{"protocol": "TCP", "port": 6443}]}]) + self.assertEqual(source, before) + self.assertEqual(seen[0]["spec"]["egress"], source["spec"]["egress"]) + self.assertEqual(seen[0]["spec"]["podSelector"], + {"matchLabels": {"kars.azure.com/e2e-policy-control": "true"}}) + self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "policy-uid", "resourceVersion": "3"}) + h.policy_connectivity_diagnostics({}, {}, deny_all=True) + self.assertEqual(seen[2]["spec"], {"policyTypes": ["Egress"], "egress": [], + "podSelector": {"matchLabels": {"kars.azure.com/e2e-policy-control": "deny-all"}}}) + self.assertEqual(source, before) + + def test_blocked_authority_diagnostics_report_source_coordinates_not_status_detail(self): + root = Path(__file__).resolve().parents[3] + cases = [ + ("Reviewed SRE consumer was changed or replaced", "migration.rs", "controller-rejection", None), + ("Private SRE role definition has excessive or unsupported authority", "bindings.rs", "controller-rejection", None), + ("Retire reviewed SRE ClusterRoleBinding: Kubernetes status 403", "bindings.rs", "kubernetes-status", 403), + ("Inspect reserved SRE token identity: Kubernetes transport/serialization failure", + "credential_guard.rs", "kubernetes-transport", None), + ] + for detail, source, category, code in cases: + with self.subTest(source=source): + result = authority_failure_site(root, detail) + self.assertTrue(result["source"].endswith(source)) + self.assertGreater(result["line"], 0) + self.assertEqual(result["category"], category) + self.assertEqual(result.get("httpStatus"), code) + self.assertNotIn(detail, json.dumps(result)) + for unsafe in (detail + MARKER, MARKER + detail, detail + "\n" + MARKER): + self.assertEqual(authority_failure_site(root, unsafe), {"category": "unclassified"}) + for unsafe in (None, {}, "", MARKER, "private body: Kubernetes status 403", + "Retire reviewed SRE ClusterRoleBinding: Kubernetes status 403 " + MARKER): + self.assertEqual(authority_failure_site(root, unsafe), {"category": "unclassified"}) + + def test_control_consumer_fixture_satisfies_required_sandbox_fields(self): + captured = [] + class Captured(Exception): + pass + def capture(obj, **_kwargs): + captured.append(obj) + raise Captured + with self.assertRaises(Captured): + seed_control_consumer(types.SimpleNamespace(create=capture)) + root = Path(__file__).resolve().parents[3] + schema = (root / "deploy/helm/kars/templates/crd.yaml").read_text() + spec = schema.split(" spec:\n", 1)[1] + required = json.loads(re.search(r"required: (\[[^\n]+\])", spec).group(1)) + for source in (captured[0], reserved_source_probe()): + with self.subTest(name=source["metadata"]["name"]): + self.assertTrue(set(required).issubset(source["spec"])) + self.assertEqual(source["spec"]["inferenceRef"], {"name": "sre-inference"}) + self.assertEqual(source["spec"]["runtime"]["kind"], "BYO") + self.assertNotIn("agent", source["spec"]) + + def test_failed_command_reports_only_source_exit_and_allowlisted_category(self): + root = Path(__file__).resolve().parent + with tempfile.TemporaryDirectory(prefix=".harness-unit-", dir=root) as folder: + harness = Harness.__new__(Harness) + harness.work = harness.root = Path(folder) + harness.deadline = time.monotonic() + 30 + harness.phase = "prepare" + process = types.SimpleNamespace(returncode=1, communicate=lambda **_kwargs: ( + MARKER, f"Error from server (Invalid): {MARKER} secret/header/kubeconfig body")) + with patch("sre_authority.common.subprocess.Popen", return_value=process): + with self.assertRaises(AssertionError) as failure: + harness.run(["kubectl", "--token", MARKER], data=MARKER) + message = str(failure.exception) + self.assertIn("harness_test.py:test_failed_command_reports_only_source_exit_and_allowlisted_category:", message) + self.assertIn("exit=1; category=Invalid", message) + for forbidden in (MARKER, "--token", "secret/header/kubeconfig body", str(harness.root)): + self.assertNotIn(forbidden, message) + + def test_secret_type_rejection_is_specific_and_not_counted_as_vap_denial(self): + cause = {"field": "type", "reason": "FieldValueInvalid", + "message": 'Invalid value: "Opaque": field is immutable'} + body = {"kind": "Status", "reason": "Invalid", "details": {"causes": [cause]}} + assert_token_type_immutable(Response(422, body), "type update") + with self.assertRaises(AssertionError): + assert_denial(Response(422, body), "type update", "kars-sre-no-legacy-tokens") + for code in (200, 400, 403, 404, 409, 500): + with self.subTest(code=code), self.assertRaises(AssertionError): + assert_token_type_immutable(Response(code, body), "type update") + for invalid in ( + {**body, "reason": "Forbidden"}, + {**body, "details": {"causes": []}}, + {**body, "details": {"causes": [{**cause, "field": "metadata.annotations"}]}}, + {**body, "details": {"causes": [{**cause, "message": "different validation failure"}]}}, + ): + with self.subTest(body=invalid), self.assertRaises(AssertionError): + assert_token_type_immutable(Response(422, invalid), "type update") + + def test_denials_require_real_forbidden_and_the_intended_policy(self): + valid = {"kind": "Status", "reason": "Forbidden", "message": "kars-sre-private-mounts denied"} + assert_denial(Response(403, valid), "probe", "kars-sre-private-mounts") + for code in (200, 201, 400, 404, 409, 422, 500): + with self.assertRaises(AssertionError): + assert_denial(Response(code, valid), "probe", "kars-sre-private-mounts") + with self.assertRaises(AssertionError): + assert_denial(Response(403, {**valid, "message": "ordinary RBAC denied"}), "probe", "kars-sre-private-mounts") + + def test_secret_watch_never_confuses_empty_success_with_authorization_denial(self): + forbidden = Response(403, {"kind": "Status", "reason": "Forbidden"}) + assert_watch_result(forbidden, False, False) + assert_watch_result(Response(200, {}), True, True) + for observed in (True, False): + with self.assertRaises(AssertionError): + assert_watch_result(Response(200, {}), observed, False) + with self.assertRaises(AssertionError): + assert_watch_result(Response(200, {}), False, True) + with self.assertRaises(AssertionError): + assert_watch_result(Response(404, {"kind": "Status", "reason": "NotFound"}), False, False) + + def test_review_arguments_pin_every_uid_resource_version_and_consumer(self): + spec = { + "controller": {}, "sandbox": {"uid": "source-uid"}, "runtimeNamespace": {"uid": "namespace-uid"}, + "legacyBindings": [ + {"kind": "ClusterRoleBinding", "name": "reader", "uid": "binding-a", "resourceVersion": "10"}, + {"kind": "RoleBinding", "namespace": "work", "name": "writer", "uid": "binding-b", "resourceVersion": "20"}, + ], + "legacyConsumer": {"uid": "consumer-uid", "resourceVersion": "30"}, + } + args = review_args(spec, {"metadata": {"uid": "registration-uid", "resourceVersion": "40"}}) + self.assertIn("ClusterRoleBinding//reader=binding-a@10", args) + self.assertIn("RoleBinding/work/writer=binding-b@20", args) + self.assertIn("consumer-uid@30", args) + self.assertEqual(args[-4:], ["--registration-uid", "registration-uid", "--resource-version", "40"]) + self.assertEqual(enrollment_json(json.dumps(spec) + "\n--binding 'review'\n"), spec) + with self.assertRaises(AssertionError): + enrollment_json('{"kind":"Status","reason":"Forbidden"}') + self.assertEqual(printed_object("Preview:\n" + json.dumps({"spec": spec}) + "\n"), + {"spec": spec}) + + def test_namespace_proof_independently_checks_full_claim_and_backlink(self): + source = {"metadata": {"name": "sre", "namespace": SYSTEM, "uid": "source-uid", "resourceVersion": "1", + "annotations": {NAMESPACE_UID: "namespace-uid"}}} + namespace = {"metadata": {"name": RUNTIME, "uid": "namespace-uid", "resourceVersion": "2", + "annotations": {CLAIM_VERSION: "v1", SOURCE_NS: SYSTEM, SOURCE_NAME: "sre", SOURCE_UID: "source-uid"}}} + core = {"metadata": {"uid": "core-uid"}} + assert_claim(source, namespace, core, "core-uid") + for key in (CLAIM_VERSION, SOURCE_NS, SOURCE_NAME, SOURCE_UID): + bad = copy.deepcopy(namespace) + bad["metadata"]["annotations"][key] = "wrong" + with self.assertRaises(AssertionError): + assert_claim(source, bad, core, "core-uid") + bad = copy.deepcopy(source) + bad["metadata"]["annotations"][NAMESPACE_UID] = "old-namespace-uid" + with self.assertRaises(AssertionError): + assert_claim(bad, namespace, core, "core-uid") + with self.assertRaises(AssertionError): + assert_claim(source, namespace, core, "old-core-uid") + + def test_real_api_status_handler_rejects_errors_without_echoing_secret_bodies(self): + harness = Harness.__new__(Harness) + seen = [] + def request(method, path, **kwargs): + seen.append((method, path, kwargs)) + return Response(403, {"message": MARKER}) + harness.client = lambda _user: types.SimpleNamespace(request=request) + with self.assertRaises(AssertionError) as failure: + harness.api("PATCH", "/api/v1/namespaces/example/secrets/example", + body={"stringData": {"token": MARKER}}, status=200) + self.assertIn("HTTP 403", str(failure.exception)) + self.assertNotIn(MARKER, str(failure.exception)) + self.assertEqual(seen[0][2]["headers"]["Content-Type"], "application/merge-patch+json") + self.assertEqual(harness.api("GET", "/missing").status_code, 403) + + def test_secret_projection_assertions_detect_all_copy_channels(self): + safe = {"kind": "Secret", "metadata": {"name": "example"}, + "data": {"operator-token": "", "password": ""}} + assert_filtered(safe) + for key in ("labels", "annotations", "managedFields"): + with self.assertRaises(AssertionError): + assert_filtered({**safe, "metadata": {"name": "example", key: {"copy": MARKER}}}) + with self.assertRaises(AssertionError): + assert_filtered({**safe, "stringData": {"copy": MARKER}}) + with self.assertRaises(AssertionError): + assert_filtered({**safe, "data": {"operator-token": MARKER, "password": ""}}) + + def test_token_probe_cannot_retain_admin_client_certificate(self): + root = Path(__file__).resolve().parent + with tempfile.TemporaryDirectory(prefix=".harness-unit-", dir=root) as folder: + harness = Harness.__new__(Harness) + harness.work = Path(folder) + harness.config = { + "clusters": [{"name": "kind-kars-e2e", "cluster": {"server": "https://127.0.0.1", "certificate-authority-data": "Y2E="}}], + "users": [{"name": "admin", "user": {"client-certificate-data": "PRIVATE", "client-key-data": "PRIVATE"}}], + } + harness.clients = {} + harness.server = "https://127.0.0.1" + harness.token_config("opaque", "not-a-real-token") + config = json.loads((harness.work / "opaque.json").read_text()) + self.assertEqual(config["users"][0]["user"], {"token": "not-a-real-token"}) + + class Context: + def __init__(self): + self.loaded = False + + def load_cert_chain(self, *_args): + self.loaded = True + + contexts = [] + def context_factory(**_kwargs): + context = Context() + contexts.append(context) + return context + clients = [] + def client_factory(**kwargs): + client = types.SimpleNamespace(**kwargs) + clients.append(client) + return client + harness.httpx = types.SimpleNamespace(Client=client_factory) + with patch("sre_authority.common.ssl.create_default_context", side_effect=context_factory): + harness.client("opaque") + harness.client("admin") + self.assertFalse(contexts[0].loaded) + self.assertTrue(contexts[1].loaded) + self.assertIsNot(contexts[0], contexts[1]) + self.assertNotIn("cert", clients[0].__dict__) + self.assertEqual(clients[0].headers["Authorization"], "Bearer not-a-real-token") + self.assertNotIn("Authorization", clients[1].headers) + + def test_legacy_seed_precedes_new_install_and_other_acceptance(self): + root = Path(__file__).resolve().parents[3] + main = (root / "tests/e2e/run.sh").read_text().split("main() {", 1)[1] + self.assertLess(main.index("prepare_sre_authority_legacy"), main.index("install_crds")) + self.assertLess(main.index("test_sre_authority_migration"), main.index("test_create_sandbox")) + fixture = (root / "tests/e2e/sre_authority/fixtures.py").read_text() + self.assertIn("8b206065608593667a40665b3f48225ef9ce278d", fixture) + self.assertIn("https://github.com/Azure/kars.git", fixture) + self.assertNotIn("--force-conflicts", fixture) + self.assertNotIn("--validate=false", fixture) + self.assertLess(fixture.index('h.state["legacy_review_before_guards"]'), + fixture.index("seed_privacy_gaps(h)")) + self.assertLess(fixture.index("seed_privacy_gaps(h)"), + fixture.index('h.cli("authority", "stage"')) + from sre_authority.common import POLICIES + self.assertIn("no-legacy-tokens", POLICIES) + + def test_staged_setup_waits_for_builtins_without_skipping_migration_readiness(self): + root = Path(__file__).resolve().parents[3] + helper = root / "tests/e2e/sre-authority.sh" + for version, expected in (("v3.16.4", "--wait"), ("v4.2.4", "--wait=legacy")): + with self.subTest(version=version): + result = subprocess.run( + ["bash", "-c", 'source "$1"; sre_migration_helm_wait_arg "$2"', + "sre-wait-test", str(helper), version], + capture_output=True, text=True, check=True, timeout=5, + ) + self.assertEqual(result.stdout.strip(), expected) + for version in ("v5.0.0", "unexpected"): + result = subprocess.run( + ["bash", "-c", 'source "$1"; sre_migration_helm_wait_arg "$2"', + "sre-wait-test", str(helper), version], + capture_output=True, text=True, check=False, timeout=5, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + script = (root / "tests/e2e/run.sh").read_text() + install = script.split("install_crds() {", 1)[1].split("\nteardown()", 1)[0] + self.assertIn("local helm_wait_arg=--wait", install) + staged = install.split('if [ "$SRE_LEGACY_PREPARED" = "1" ]; then', 1)[1].split("\n fi", 1)[0] + self.assertIn('sre_migration_helm_wait_arg "$helm_version"', staged) + self.assertIn('"$helm_wait_arg" --timeout 5m', install) + main = script.split("main() {", 1)[1] + self.assertIn(" install_crds\n", main) + self.assertIn(" test_sre_authority_migration\n", main) + self.assertLess(main.index("install_crds"), main.index("test_sre_authority_migration")) + self.assertLess(main.index("test_sre_authority_migration"), main.index("test_create_sandbox")) + + def test_kind_prerequisites_do_not_remove_the_rust_httpx_test_step(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + self.assertIn("Legacy Hermes HTTPS client test dependency", workflow) + kind = workflow.split(" e2e-kind:", 1)[1].split(" bench-regression:", 1)[0] + self.assertIn("actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", kind) + self.assertIn("npm ci && npm run build", kind) + self.assertIn("'httpx==0.28.1'", kind) + self.assertGreaterEqual(workflow.count("'httpx==0.28.1'"), 2) + + def test_kind_runtime_filter_includes_all_shared_security_modules(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + kind = workflow.split(" e2e-kind:", 1)[1].split(" bench-regression:", 1)[0] + expression = re.search(r"\| grep -E '([^']+)'", kind) + self.assertIsNotNone(expression) + pattern = expression.group(1) + for path in ("shared/sre_privacy.rs", "shared/another_security_module.rs", + "controller/src/sre_authority.rs", "cli/src/lib/sre-authority.ts"): + with self.subTest(path=path): + self.assertRegex(path, pattern) + for path in ("docs/how-to/sre-authority.md", "unrelated/shared/sre_privacy.rs"): + with self.subTest(path=path): + self.assertNotRegex(path, pattern) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/legacy_crd_probe.py b/tests/e2e/sre_authority/legacy_crd_probe.py new file mode 100644 index 000000000..c5fc5589f --- /dev/null +++ b/tests/e2e/sre_authority/legacy_crd_probe.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Same-Kind historical Helm wait/upgrade proof without controller execution.""" + +from pathlib import Path +import time +import types + +from sre_authority.bootstrap_probe import converted_objects +from sre_authority.common import CONTEXT, Harness, SYSTEM, require +from sre_authority.fixtures import LEGACY_COMMIT, install_historical_chart +from sre_authority.registration_schema import ( + create_registration_crd, kind_proxy, request, write_report, +) + + +def exercise(root): + h = Harness.__new__(Harness) + h.root, h.work = root, root / ".e2e-legacy-helm" + h.work.mkdir(mode=0o700) + h.deadline, h.phase = time.monotonic() + 300, "legacy-helm-proof" + with kind_proxy(root) as (port, version): + def api(method, path, *, body=None, status=None): + code, obj = request(port, method, path, body) + if status is not None: + require(code in (status if isinstance(status, tuple) else (status,)), + f"Historical Helm API proof: HTTP {code}") + return types.SimpleNamespace(status_code=code, json=lambda: obj) + h.api = api + install_historical_chart(h) + rendered = h.run(["helm", "template", "kars", str(root / "deploy/helm/kars"), + "--namespace", SYSTEM, "--show-only", "templates/crd-karssreregistration.yaml"]) + objects = converted_objects(h.k("create", "--dry-run=client", "--validate=strict", + "-f", "-", "-o", "json", data=rendered)) + require(len(objects) == 1, "Historical Helm proof requires one registration CRD") + obj = objects[0] + obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" + obj["metadata"]["annotations"] = { + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM} + create_registration_crd(h, obj) + h.k("wait", "--for=condition=Established", "crd/karssreregistrations.kars.azure.com", + "--timeout=60s", timeout=70) + h.run(["helm", "--kube-context", CONTEXT, "upgrade", "kars", str(root / "deploy/helm/kars"), + "--namespace", SYSTEM, "--reset-then-reuse-values", "--set", "sre.authorityStage=true", + "--dry-run=server"], timeout=90) + write_report(root, "legacy-helm-readiness.json", { + "apiServer": version, "legacyCommit": LEGACY_COMMIT, + "historicalInstallAndPostInstallHook": "passed", "currentAuthorityServerDryRun": "passed", + "controllerReplicas": 0, "legacyCRDs": 18, "crdCreation": "native-Helm-only"}) + + +if __name__ == "__main__": + try: + exercise(Path(__file__).resolve().parents[3]) + except Exception as error: + detail = str(error) if isinstance(error, AssertionError) else type(error).__name__ + print(f"SRE-LEGACY-HELM-FAIL {detail}", flush=True) + raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/legacy_crds.py b/tests/e2e/sre_authority/legacy_crds.py new file mode 100644 index 000000000..1afc669c7 --- /dev/null +++ b/tests/e2e/sre_authority/legacy_crds.py @@ -0,0 +1,111 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Read-only validation of CRDs from the immutable historical Helm fixture.""" + +from .bootstrap_probe import converted_objects +from .common import SYSTEM, require +from .registration_schema import CRD_NAME, CRD_PATH + +# This is the complete historical inventory, not the current chart's CRDs. +CRDS = { + "crd-a2aagent.yaml": ("a2aagents", "A2AAgent", "Namespaced"), + "crd-egressapproval.yaml": ("egressapprovals", "EgressApproval", "Namespaced"), + "crd-inferencepolicy.yaml": ("inferencepolicies", "InferencePolicy", "Namespaced"), + "crd-karsapproval.yaml": ("karsapprovals", "KarsApproval", "Namespaced"), + "crd-karsauthconfig.yaml": ("karsauthconfigs", "KarsAuthConfig", "Cluster"), + "crd-karseval.yaml": ("karsevals", "KarsEval", "Namespaced"), + "crd-karsmemory.yaml": ("karsmemories", "KarsMemory", "Namespaced"), + "crd-karsprofile.yaml": ("karsprofiles", "KarsProfile", "Namespaced"), + "crd-karsreceipt.yaml": ("karsreceipts", "KarsReceipt", "Namespaced"), + "crd-karsskill.yaml": ("karsskills", "KarsSkill", "Namespaced"), + "crd-karssreaction.yaml": ("karssreactions", "KarsSREAction", "Namespaced"), + "crd-karstask.yaml": ("karstasks", "KarsTask", "Namespaced"), + "crd-karsteam.yaml": ("karsteams", "KarsTeam", "Namespaced"), + "crd-mcpserver.yaml": ("mcpservers", "McpServer", "Namespaced"), + "crd-toolpolicy.yaml": ("toolpolicies", "ToolPolicy", "Namespaced"), + "crd-trustgraph.yaml": ("trustgraphs", "TrustGraph", "Cluster"), + "crd.yaml": ("karssandboxes", "KarsSandbox", "Namespaced"), +} +# The historical crd.yaml contains both KarsSandbox and KarsPairing. +IDENTITIES = (*CRDS.values(), ("karspairings", "KarsPairing", "Namespaced")) + + +def validate_rendered_crds(rendered, historical): + require(isinstance(historical, dict) and set(historical) == set(CRDS), + "Historical CRD inventory differs from the immutable baseline") + require(isinstance(rendered, list) and len(rendered) == len(IDENTITIES), + "Historical render omitted or added resources") + expected = {} + for filename, identity in CRDS.items(): + identities = [identity, IDENTITIES[-1]] if filename == "crd.yaml" else [identity] + require(isinstance(historical[filename], list) and len(historical[filename]) == len(identities), + "Historical template has an unexpected number of CRDs") + for obj, expected_identity in zip(historical[filename], identities): + validate_historical_crd(obj, expected_identity) + expected[obj["metadata"]["name"]] = obj + seen = set() + for obj in rendered: + require(isinstance(obj, dict), "Historical render contained a non-object") + name = obj.get("metadata", {}).get("name") + require(name in expected and name not in seen and obj == expected[name], + "Rendered CRD content differs from its immutable historical template") + seen.add(name) + return rendered + + +def validate_historical_crd(obj, identity): + plural, kind, scope = identity + name = f"{plural}.kars.azure.com" + require(isinstance(obj, dict) and set(obj) == {"apiVersion", "kind", "metadata", "spec"} + and obj["apiVersion"] == "apiextensions.k8s.io/v1" + and obj["kind"] == "CustomResourceDefinition" + and obj["metadata"].get("name") == name + and not set(obj["metadata"]) - {"name", "labels"} + and obj["spec"].get("group") == "kars.azure.com" + and obj["spec"].get("scope") == scope + and obj["spec"].get("names", {}).get("plural") == plural + and obj["spec"].get("names", {}).get("kind") == kind, + "Historical CRD identity or ownership differs from the immutable baseline") + versions = obj["spec"].get("versions", []) + require(len(versions) == 1 and versions[0].get("name") == "v1alpha1" + and versions[0].get("served") is True and versions[0].get("storage") is True + and versions[0].get("schema", {}).get("openAPIV3Schema", {}).get("type") == "object", + "Historical CRD version/schema differs from the immutable baseline") + + +def render_legacy_crds(h, chart, sources): + require(set(sources) == set(CRDS) + and {path.name for path in (chart / "templates").glob("crd*.yaml")} == set(CRDS), + "Historical archive contains an unexpected CRD inventory") + for filename, content in sources.items(): + require("{{" not in content and (chart / "templates" / filename).read_text() == content, + "Historical CRD template changed after immutable archive extraction") + args = ["helm", "template", "kars", str(chart), "--namespace", SYSTEM, + "--set", "controller.replicas=0", "--set", "sre.enabled=false"] + for filename in CRDS: + args += ["--show-only", f"templates/{filename}"] + rendered = h.run(args, timeout=45) + def convert(text): + return converted_objects(h.k("create", "--dry-run=client", "--validate=strict", + "-f", "-", "-o", "json", data=text)) + original = convert("\n---\n".join(sources[filename] for filename in CRDS)) + require(len(original) == len(IDENTITIES), "Immutable historical CRD count changed") + historical, offset = {}, 0 + for filename in CRDS: + count = 2 if filename == "crd.yaml" else 1 + historical[filename] = original[offset:offset + count] + offset += count + return validate_rendered_crds(convert(rendered), historical) + + +def preflight_legacy_crds(h, chart, sources): + objects = render_legacy_crds(h, chart, sources) + # Only Helm creates these objects, retaining its native SSA field ownership. + # An existing object, even one claiming this release, is never adopted. + for name in [CRD_NAME] + [obj["metadata"]["name"] for obj in objects]: + response = h.api("GET", f"{CRD_PATH}/{name}", status=404) + body = response.json() + require(body.get("kind") == "Status" and body.get("reason") == "NotFound", + "Historical CRD absence was not a Kubernetes NotFound") + h.passed("All 18 historical CRDs match the immutable archive and are absent before Helm-owned creation") diff --git a/tests/e2e/sre_authority/legacy_crds_test.py b/tests/e2e/sre_authority/legacy_crds_test.py new file mode 100644 index 000000000..41ec4bffc --- /dev/null +++ b/tests/e2e/sre_authority/legacy_crds_test.py @@ -0,0 +1,144 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure historical fixture checks, not a substitute for hosted Kind acceptance.""" + +import copy +from pathlib import Path +import types +import unittest +from unittest.mock import Mock, patch + +from sre_authority.legacy_crds import ( + CRDS, IDENTITIES, preflight_legacy_crds, render_legacy_crds, validate_rendered_crds, +) +from sre_authority.registration_schema import CRD_NAME + + +def historical_objects(): + def obj(plural, kind, scope): + return { + "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", + "metadata": {"name": f"{plural}.kars.azure.com", "labels": {"app.kubernetes.io/name": "kars"}}, + "spec": {"group": "kars.azure.com", "scope": scope, "names": {"plural": plural, "kind": kind}, + "versions": [{"name": "v1alpha1", "served": True, "storage": True, + "schema": {"openAPIV3Schema": {"type": "object"}}}]}, + } + result = {filename: [obj(*identity)] for filename, identity in CRDS.items()} + result["crd.yaml"].append(obj(*IDENTITIES[-1])) + return result + + +def flattened(historical): + return [obj for objects in historical.values() for obj in objects] + + +class LegacyCRDTests(unittest.TestCase): + def test_render_requires_exact_historical_content_not_just_a_matching_name(self): + historical = historical_objects() + rendered = flattened(copy.deepcopy(historical)) + self.assertEqual(validate_rendered_crds(rendered, historical), rendered) + rendered[0]["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["description"] = "changed" + with self.assertRaises(AssertionError): + validate_rendered_crds(rendered, historical) + + def test_current_registration_extra_missing_duplicate_and_foreign_resources_are_rejected(self): + historical = historical_objects() + valid = flattened(copy.deepcopy(historical)) + registration = copy.deepcopy(valid[0]) + registration["metadata"]["name"] = CRD_NAME + for rendered in (valid[:-1], valid + [registration], [registration] + valid[1:], + [valid[0]] + valid[:-1], [{"kind": "Secret"}] + valid[1:]): + with self.subTest(count=len(rendered)), self.assertRaises(AssertionError): + validate_rendered_crds(rendered, historical) + with self.assertRaises(AssertionError): + validate_rendered_crds(valid, {**historical, "crd-karssreregistration.yaml": registration}) + + def test_historical_identity_schema_and_preexisting_ownership_are_strict(self): + def changed(section, key, value): + obj = historical_objects() + obj["crd.yaml"][0][section][key] = value + return obj + invalid = [ + changed("metadata", "name", "other.kars.azure.com"), + changed("metadata", "annotations", {"meta.helm.sh/release-name": "other"}), + changed("metadata", "ownerReferences", [{"uid": "foreign"}]), + changed("spec", "group", "foreign.example"), + changed("spec", "scope", "Cluster"), + changed("spec", "names", {"plural": "karssandboxes", "kind": "Secret"}), + changed("spec", "versions", []), + ] + for historical in invalid: + with self.subTest(historical=historical["crd.yaml"]), self.assertRaises(AssertionError): + validate_rendered_crds(flattened(historical), historical) + + def test_changed_extracted_source_and_unexpected_inventory_fail_before_render(self): + h = Mock() + class Chart: + def __truediv__(self, _name): + return self + + def glob(self, _pattern): + return [types.SimpleNamespace(name=filename) for filename in CRDS] + + def read_text(self): + return "changed" + for sources in ({filename: "original" for filename in CRDS}, + {filename: "{{ templated }}" for filename in CRDS}, + {"crd-karssreregistration.yaml": "unexpected"}): + with self.assertRaises(AssertionError): + render_legacy_crds(h, Chart(), sources) + h.run.assert_not_called() + h.k.assert_not_called() + + def test_every_absence_is_read_only_before_native_helm_creates_any_crd(self): + h = Mock() + h.api.return_value = types.SimpleNamespace(json=lambda: {"kind": "Status", "reason": "NotFound"}) + objects = flattened(historical_objects()) + with patch("sre_authority.legacy_crds.render_legacy_crds", return_value=objects): + preflight_legacy_crds(h, Path("unused-test-chart"), {}) + names = [CRD_NAME] + [obj["metadata"]["name"] for obj in objects] + self.assertEqual(len(names), 19) + self.assertEqual([call.args[1].rsplit("/", 1)[-1] for call in h.api.call_args_list], names) + self.assertTrue(all(call.args[0] == "GET" and call.kwargs == {"status": 404} + for call in h.api.call_args_list)) + h.create.assert_not_called() + h.k.assert_not_called() + h.passed.assert_called_once() + + def test_existing_or_inaccessible_crd_aborts_without_adoption_or_retry(self): + for code in (200, 401, 403, 409, 500): + h = Mock() + h.api.side_effect = AssertionError(f"HTTP {code}") + with patch("sre_authority.legacy_crds.render_legacy_crds", + return_value=flattened(historical_objects())): + with self.subTest(code=code), self.assertRaisesRegex(AssertionError, f"HTTP {code}"): + preflight_legacy_crds(h, Path("unused-test-chart"), {}) + h.api.assert_called_once() + h.create.assert_not_called() + h.passed.assert_not_called() + + def test_false_not_found_is_not_absence_proof(self): + for body in ({"kind": "Secret", "reason": "NotFound"}, {"kind": "Status", "reason": "Forbidden"}): + h = Mock() + h.api.return_value = types.SimpleNamespace(json=lambda: body) + with patch("sre_authority.legacy_crds.render_legacy_crds", + return_value=flattened(historical_objects())), self.assertRaises(AssertionError): + preflight_legacy_crds(h, Path("unused-test-chart"), {}) + h.create.assert_not_called() + h.passed.assert_not_called() + + def test_initial_helm_uses_existing_versioned_waiter_without_custom_creation(self): + fixture = Path(__file__).with_name("fixtures.py").read_text() + install = fixture.split("def install_historical_chart(h):", 1)[1].split("def prepare_legacy(h):", 1)[0] + self.assertLess(install.index("preflight_legacy_crds(h,"), install.index('h.run(["helm", "install"')) + self.assertIn('sre_migration_helm_wait_arg "$2"', install) + self.assertIn('wait_arg, "--timeout", "120s"', install) + self.assertLess(install.index('h.run(["helm", "install"'), install.index('h.get("toolpolicy"')) + self.assertNotIn("h.create(", install) + for bypass in ("--take-ownership", "--force", "--validate=false", "--no-hooks", "governance.enabled=false"): + self.assertNotIn(bypass, fixture) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/migration.py b/tests/e2e/sre_authority/migration.py new file mode 100644 index 000000000..927769a55 --- /dev/null +++ b/tests/e2e/sre_authority/migration.py @@ -0,0 +1,284 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json + +from .common import ( + AGENT, EPOCH, OPERATORS, PRIVATE, RUNTIME, SYSTEM, TENANT, + assert_claim, assert_denial, enrollment_json, fingerprint, printed_object, require, review_args, +) +from .fixtures import CONTROL, CONTROL_NS, GROUP_BINDING, delegate_operators +from .admission import admission_cases, policies_ready +from .credential_paths import ( + assert_fresh_private_identity, block_prestaged_paths, fixture_review, secret_watch, token_secret_denials, +) + + +def cli_failure(result, text, label): + require(result.returncode != 0 and text in (result.stdout + result.stderr), + f"{label}: command did not fail for the intended reason") + + +def pods(h, namespace): + return json.loads(h.k("get", "pods", "-n", namespace, "-o", "json"))["items"] + + +def snapshot_grants(h, spec): + return {binding["name"]: h.get(binding["kind"].lower(), binding["name"], binding.get("namespace")) + for binding in spec["legacyBindings"]} + + +def running_controller_stage(h): + controller = h.get("deployment", "kars-controller", SYSTEM) + require(controller and controller["spec"].get("replicas", 0) > 0 + and controller.get("status", {}).get("availableReplicas", 0) > 0, + "Staging compatibility requires a real running controller, not replicas=0") + require(h.get("karssreregistrations.kars.azure.com", "canonical") is None, + "Staging compatibility must run before enrollment") + def unenrolled(): + source = h.get("karssandbox", "sre", SYSTEM) + return source if source and any(c.get("type") == "Ready" and c.get("status") == "False" + for c in source.get("status", {}).get("conditions", [])) else False + source = h.poll("live controller observes unenrolled source as not Ready", unenrolled, seconds=60) + before = h.get("clusterrolebinding", "kars-sre-reader") + h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", timeout=180) + after = h.get("clusterrolebinding", "kars-sre-reader") + require(before["metadata"]["uid"] == after["metadata"]["uid"] and before["subjects"] == after["subjects"], + "Running-controller stage changed legacy grants") + require(h.get("karssandbox", "sre", SYSTEM)["metadata"]["uid"] == source["metadata"]["uid"], + "Running-controller stage replaced the source") + require(h.get("karssreregistrations.kars.azure.com", "canonical") is None + and h.get("secret", PRIVATE, RUNTIME) is None, + "Running-controller stage enrolled or issued private material") + policies_ready(h) + h.passed("Actual CLI stage returns with a running controller and unenrolled Ready=False source; full migration Ready remains mandatory") + + +def legacy_migration(h): + policies_ready(h) + running_controller_stage(h) + delegate_operators(h) + # Group grants were deliberately created before admission, never smuggled + # through the new deny policies. + original = h.get("clusterrolebinding", "kars-sre-reader") + cli_failure(h.cli("authority", "preview", user="registrar", expected=None), + "broad group grant", "broad-group review") + require(h.get("clusterrolebinding", "kars-sre-reader")["subjects"] == original["subjects"], + "Group ambiguity mutated direct legacy subjects") + require(h.get("secret", PRIVATE, RUNTIME) is None, "Private material appeared before explicit enrollment") + h.k("delete", "clusterrolebinding", GROUP_BINDING, "--wait=true", "--timeout=30s") + h.passed("Actual preview blocks an unsafe pre-existing group grant before direct grants or credentials change") + cli_failure(h.cli("authority", "preview", user="registrar", expected=None), + "broad group grant", "watch-only broad-group review") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", + user="watch-only"), "watch-only principal GET") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets", + user="watch-only"), "watch-only principal LIST") + secret_watch(h, CONTROL_NS, "e2e-watch-only-before", user="watch-only", allowed=True) + h.passed("Watch-only group member is denied Secret GET/LIST but genuinely observes a newly created dummy Secret; preview blocks the grant") + + h.k("rollout", "status", f"deployment/{CONTROL}", "-n", CONTROL_NS, "--timeout=150s", timeout=160) + spec = fixture_review(h) + require(spec["sandbox"]["uid"] == h.state["legacy_source_uid"], "Staging adopted a replacement SRE source") + require(spec["runtimeNamespace"]["uid"] == h.state["legacy_namespace_uid"], "Staging replaced the runtime namespace") + assert_claim(h.get("karssandbox", "sre", SYSTEM), h.get("namespace", RUNTIME), + h.get("namespace", SYSTEM), h.state["system_uid"]) + require(spec.get("legacyConsumer") and len(spec["legacyBindings"]) >= 2, "Legacy reviews are incomplete") + h.state["old_agent_sa_uid"] = h.get("serviceaccount", "sandbox", RUNTIME)["metadata"]["uid"] + old_pods = {pod["metadata"]["uid"] for pod in pods(h, RUNTIME)} + old_control_pods = {pod["metadata"]["uid"] for pod in pods(h, CONTROL_NS)} + require(old_pods and old_control_pods, "Legacy/control consumers are not actually running") + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="old-agent", status=200) + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets", user="old-agent", status=200) + before = snapshot_grants(h, spec) + admission_cases(h, spec) + token_secret_denials(h, before_enrollment=True) + # The controller rejects the pre-existing token alias first, then the + # watch-only group grant. Only explicit UID/RV-fenced fixture cleanup + # removes them; the registration is left blocked on an incomplete review. + blocked = block_prestaged_paths(h, spec, before) + cli_failure(h.cli("authority", "enroll", "--sandbox-uid", spec["sandbox"]["uid"], + "--namespace-uid", spec["runtimeNamespace"]["uid"], + user="registrar", expected=None), "Every legacy binding", "missing binding reviews") + current = h.get("karssreregistrations.kars.azure.com", "canonical") + require(current["metadata"]["uid"] == blocked["metadata"]["uid"] and current["spec"] == blocked["spec"] + and current["metadata"]["generation"] == blocked["metadata"]["generation"], + "Unreviewed CLI enrollment changed the blocked registration") + require(snapshot_grants(h, spec) == before, "Unreviewed enrollment mutated legacy bindings") + require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, "Unreviewed enrollment stopped the consumer") + h.passed("Unreviewed bindings/consumer cannot enroll or mutate an occupied legacy source") + require(snapshot_grants(h, spec) == before, "Controller mutated grants before validating the entire review set") + require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, "Controller stopped an unreviewed consumer") + require(h.get("secret", PRIVATE, RUNTIME) is None, "Controller issued private material for an incomplete review") + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="old-agent", status=200) + h.passed("Real registration/controller rejects an incomplete review before grant retirement, consumer stop or private issuance") + + # Refresh once after read-only tests; use the exact current RVs, not guessed + # names or stale values from the fixture bootstrap. + spec = enrollment_json(h.cli("authority", "preview", user="registrar")) + reviewed = review_args(spec, current) + preview = h.cli("authority", "enroll", *reviewed, "--dry-run", user="registrar") + require(printed_object(preview)["spec"] == spec, "Enrollment dry-run differs from reviewed API identities") + after = h.get("karssreregistrations.kars.azure.com", "canonical") + require(after["spec"] == current["spec"] and after["metadata"]["generation"] == current["metadata"]["generation"] + and after["metadata"]["uid"] == current["metadata"]["uid"], "Enrollment dry-run mutated registration") + reviewed = review_args(spec, after) + h.cli("authority", "enroll", *reviewed, user="registrar") + require(h.get("karssreregistrations.kars.azure.com", "canonical")["spec"] == spec, + "Persisted enrollment differs from the exact reviewed identities") + + issuance_seen = False + def migrated(): + nonlocal issuance_seen + private = h.get("secret", PRIVATE, RUNTIME) + if private and "kube-token" in private.get("data", {}) and not issuance_seen: + # Real held bearer token, with no client certificate fallback. + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", + user="old-agent"), "old held principal GET before private issuance") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets", + user="old-agent"), "old held principal LIST before private issuance") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets?watch=true&timeoutSeconds=1", + user="old-agent"), "old held principal WATCH before private issuance") + require(not old_pods.intersection(pod["metadata"]["uid"] for pod in pods(h, RUNTIME)), + "Private credentials issued while a reviewed old consumer still exists") + issuance_seen = True + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + require(reg is not None, "Enrollment disappeared during migration") + status = reg.get("status", {}) + require(not (status.get("phase") == "Blocked" and status.get("observedGeneration") == reg["metadata"]["generation"]), + "Reviewed SRE migration was blocked") + return reg if status.get("phase") == "Ready" and status.get("observedGeneration") == reg["metadata"]["generation"] else False + reg = h.poll("reviewed migration and credential-order proof", migrated, seconds=240, interval=0.25) + require(issuance_seen, "No private issuance was observed") + assert_fresh_private_identity(h, reg) + h.passed("Controller-created private ServiceAccount has a new registered UID, distinct from the operator-deleted unowned same-name fixture") + h.cli("authority", "migrate", user="registrar", timeout=45) + h.wait_ready() + h.passed("Real old-principal GET/LIST/WATCH denial and old Pod termination precede private credential issuance") + for namespace, cluster, name in [ + (CONTROL_NS, False, "e2e-old-watch-control"), + (RUNTIME, False, "e2e-old-watch-private"), + (CONTROL_NS, True, "e2e-old-watch-cluster"), + ]: + secret_watch(h, namespace, name, cluster=cluster) + h.passed("Held legacy token receives actual HTTP Forbidden for namespaced and cluster Secret watches, including newly created dummy-Secret probes") + require(h.get("serviceaccount", "sandbox", RUNTIME)["metadata"]["uid"] == h.state["old_agent_sa_uid"], + "Migration used ServiceAccount recreation instead of revoking held-principal authorization") + reader = h.get("clusterrolebinding", "kars-sre-reader") + require(h.state["unrelated_subject"] in reader["subjects"], "Migration removed an unrelated binding subject") + require(not any(subject.get("name") == "sandbox" and subject.get("namespace") == RUNTIME for subject in reader["subjects"]), + "Legacy SRE subject remains bound") + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="unrelated", status=200) + h.passed("Unrelated binding subjects remain authorized; the old Sandbox ServiceAccount UID is preserved but denied") + + rotated = h.get("secret", "router-services-admin", CONTROL_NS) + require(rotated["metadata"]["uid"] == h.state["control_secret_uid"], "Owned control Secret was replaced instead of rotated") + require(fingerprint(rotated) != h.state["control_digest"], "Owned exposed control token was not rotated") + current_pods = pods(h, CONTROL_NS) + require(current_pods and not old_control_pods.intersection(pod["metadata"]["uid"] for pod in current_pods), + "A startup-cached old control-token consumer survived Ready") + expected = fingerprint(rotated) + for pod in current_pods: + require(pod["metadata"].get("annotations", {}).get(EPOCH) == reg["status"]["privacyEpoch"], + "Control consumer lacks the new privacy epoch") + cached = h.k("exec", "-n", CONTROL_NS, pod["metadata"]["name"], "-c", "credential-consumer", + "--", "cat", "/proof/cached.sha", timeout=25).split()[0] + require(cached == expected, "Restarted control consumer did not cache the rotated token") + h.passed("Owned router-services-admin token rotates and every cached old consumer is gone before Ready") + h.cli("install", timeout=240) + rollback_guard(h) + h.state["legacy_registration_uid"] = reg["metadata"]["uid"] + h.save() + + +def rollback_guard(h): + # Execute the actual CLI guard against this API, without pretending a Kind + # cluster has an Azure upgrade context or mocking an Azure command. + script = """ +const {pathToFileURL}=require('node:url'); +const {execa}=await import(pathToFileURL(process.cwd()+'/cli/node_modules/execa/index.js')); +const {assertRollbackSafe}=await import(pathToFileURL(process.cwd()+'/cli/dist/lib/sre-authority.js')); +const execute=(file,args,options)=>execa(file,['--context','kind-kars-e2e',...args],options); +try { await assertRollbackSafe(execute); process.exitCode=9; } +catch(error) { if(!String(error.message).includes('Rollback across enrolled SRE authority is unsafe')) throw error; } +""" + # CommonJS eval cannot use top-level await; the async wrapper keeps imports + # on the actual compiled CLI module, not an alternate acceptance model. + h.run(["node", "-e", f"(async()=>{{{script}}})().catch(()=>{{process.exitCode=1}})"], timeout=30) + h.passed("Actual CLI rollback guard rejects regranting a legacy release while enrollment audit history exists") + + +def retire_and_uninstall(h): + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + h.cli("authority", "retire", "--registration-uid", reg["metadata"]["uid"], + "--resource-version", reg["metadata"]["resourceVersion"], user="registrar", timeout=240) + retired = h.get("karssreregistrations.kars.azure.com", "canonical") + require(retired["status"]["phase"] == "Retired" and retired["spec"]["enabled"] is False, + "Retirement did not complete") + for name in (PRIVATE, AGENT): + require(h.get("secret", name, RUNTIME) is None, "Private credential survived retirement") + for kind in ("clusterrolebindings", "rolebindings"): + items = json.loads(h.k("get", kind, "-A", "-o", "json"))["items"] + require(not any(item["metadata"].get("annotations", {}).get("kars.azure.com/sre-registration-uid") == reg["metadata"]["uid"] + for item in items), "Owned private grant survived retirement") + h.cli("uninstall", timeout=180) + h.poll("SRE source removal", lambda: h.get("karssandbox", "sre", SYSTEM) is None, seconds=120) + h.poll("SRE namespace removal", lambda: h.get("namespace", RUNTIME) is None, seconds=120) + require(h.get("namespace", SYSTEM)["metadata"]["uid"] == h.state["system_uid"], "Retirement changed core namespace identity") + retained = h.get("karssreregistrations.kars.azure.com", "canonical") + require(retained["metadata"]["uid"] == reg["metadata"]["uid"] and retained["status"]["phase"] == "Retired", + "Retired registration audit record was removed or replaced") + h.passed("Retire/uninstall revoke owned grants/credentials and remove the source namespace while retaining core UID and audit registration") + + +def fresh_reenrollment(h): + policies_ready(h) + retained = h.get("karssreregistrations.kars.azure.com", "canonical") + require(retained and retained["status"]["phase"] == "Retired", "Fresh phase requires retained retired history") + foreign = h.create({"apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": RUNTIME, "labels": {"e2e-foreign-occupant": "true"}}}) + h.create({"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "foreign-sentinel", "namespace": RUNTIME}, + "data": {"value": "must-remain"}}) + cli_failure(h.cli("authority", "stage-source", expected=None), + "Existing kars-sre namespace", "foreign occupant") + require(h.get("namespace", RUNTIME)["metadata"]["uid"] == foreign["metadata"]["uid"], "Foreign namespace was replaced") + require(h.get("configmap", "foreign-sentinel", RUNTIME)["data"]["value"] == "must-remain", "Foreign content was mutated") + require(h.get("karssandbox", "sre", SYSTEM) is None and h.get("secret", PRIVATE, RUNTIME) is None, + "Foreign occupancy acquired a source or private privilege") + h.namespace_delete(RUNTIME, foreign["metadata"]["uid"]) + h.passed("Foreign runtime occupancy blocks fresh staging before source creation or private issuance") + + created = h.cli("authority", "stage-source") + spec = enrollment_json(h.cli("authority", "preview", user="registrar")) + assert_claim(h.get("karssandbox", "sre", SYSTEM), h.get("namespace", RUNTIME), + h.get("namespace", SYSTEM), h.state["system_uid"]) + require(f'--sandbox-uid {spec["sandbox"]["uid"]}' in created + and f'--namespace-uid {spec["runtimeNamespace"]["uid"]}' in created, + "Fresh staging did not return the actual CREATE/claim UIDs") + require(spec["sandbox"]["uid"] != h.state["legacy_source_uid"] and + spec["runtimeNamespace"]["uid"] != h.state["legacy_namespace_uid"], "Fresh source reused old authority identities") + cli_failure(h.cli("authority", "stage-source", expected=None), "SRE source already exists", "source adoption race") + cli_failure(h.cli("authority", "enroll", "--sandbox-uid", h.state["legacy_source_uid"], + "--namespace-uid", h.state["legacy_namespace_uid"], user="registrar", expected=None), + "no longer matches", "stale source UID enrollment") + retained = h.get("karssreregistrations.kars.azure.com", "canonical") + reviewed = review_args(spec, retained) + dry = printed_object(h.cli("authority", "enroll", *reviewed, "--dry-run", user="registrar")) + require(dry["spec"] == spec, "Fresh enrollment differs from its reviewed identities") + h.cli("authority", "enroll", *reviewed, user="registrar") + require(h.get("karssreregistrations.kars.azure.com", "canonical")["spec"] == spec, + "Fresh CAS re-enrollment retained unreviewed identities from the retired registration") + h.cli("authority", "migrate", user="registrar", timeout=240) + h.wait_ready() + h.cli("install", timeout=240) + # Recreate only our test Role/Bindings in the newly claimed namespace. + role = h.get("role", "e2e-sre-admission-probe", TENANT) + h.create({"apiVersion": role["apiVersion"], "kind": "Role", + "metadata": {"name": role["metadata"]["name"], "namespace": RUNTIME}, "rules": role["rules"]}) + for account, namespace in (("tenant", TENANT), ("registrar", OPERATORS)): + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": f"e2e-sre-{account}-runtime", "namespace": RUNTIME}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": role["metadata"]["name"]}, + "subjects": [{"kind": "ServiceAccount", "name": account, "namespace": namespace}]}) + h.passed("Fresh atomic source/claim and delegated CAS re-enrollment reach Ready without adopting stale UIDs") diff --git a/tests/e2e/sre_authority/network_diagnostics.py b/tests/e2e/sre_authority/network_diagnostics.py new file mode 100644 index 000000000..91950ca4c --- /dev/null +++ b/tests/e2e/sre_authority/network_diagnostics.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Throwaway no-credential network controls, never mutations of the SRE guard.""" + +import copy +import json +import re + +from .common import STANDIN, require + + +def guard_variant(root, pod, variant): + require(variant in ("full", "filter-only", "legacy-full"), "Unknown guard comparison") + source = (root / "controller/src/reconciler/pod_spec.rs").read_text() + body = source.split("pub(crate) fn build_egress_guard_command", 1)[1].split("\n cmd\n", 1)[0] + literals = re.findall(r'cmd\.push_str\(\s*("(?:\\.|[^"\\])*")\s*\)', body) + require(len(literals) == 8, "Guard comparison requires the exact reviewed command structure") + expected = "".join(json.loads(literal) for literal in literals) + require(not any(char in expected for char in ("$", "`", ";", "\n")), + "Guard comparison refuses dynamic shell constructs") + matches = [item for item in pod["spec"].get("initContainers", []) if item["name"] == "egress-guard"] + require(len(matches) == 1, "Guard comparison requires exactly one actual egress guard") + original = matches[0] + require(original["image"] == STANDIN and original["command"] == ["sh", "-c", expected] + and not any(original.get(key) for key in ("env", "envFrom", "volumeMounts")), + "Actual guard differs from the reviewed source or contains extra inputs") + script = expected + if variant == "filter-only": + script = " && ".join(part for part in script.split(" && ") if not part.startswith("iptables -t nat ")) + require("iptables -A OUTPUT -m owner --uid-owner 1000 -j DROP" in script, + "Filter comparison must retain the full agent DROP boundary") + elif variant == "legacy-full": + script = re.sub(r"\biptables(?= )", "iptables-legacy", script) + script = "command -v iptables-legacy >/dev/null || exit 42; " + script + init = {key: copy.deepcopy(value) for key, value in original.items() + if key in ("name", "image", "imagePullPolicy", "command", "securityContext", "resources")} + init["command"] = ["sh", "-c", script] + return init diff --git a/tests/e2e/sre_authority/network_diagnostics_test.py b/tests/e2e/sre_authority/network_diagnostics_test.py new file mode 100644 index 000000000..36b5c116e --- /dev/null +++ b/tests/e2e/sre_authority/network_diagnostics_test.py @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +from pathlib import Path +import re +import unittest + +from sre_authority.common import STANDIN +from sre_authority.network_diagnostics import guard_variant + + +class NetworkDiagnosticTests(unittest.TestCase): + def fixture(self): + root = Path(__file__).resolve().parents[3] + source = (root / "controller/src/reconciler/pod_spec.rs").read_text() + body = source.split("pub(crate) fn build_egress_guard_command", 1)[1].split("\n cmd\n", 1)[0] + expected = "".join(json.loads(value) for value in re.findall(r'cmd\.push_str\(\s*("(?:\\.|[^"\\])*")\s*\)', body)) + return root, {"spec": {"initContainers": [{"name": "egress-guard", "image": STANDIN, + "command": ["sh", "-c", expected], "securityContext": {"runAsUser": 0}}]}} + + def test_full_control_reproduces_actual_guard_without_mutating_it(self): + root, pod = self.fixture() + original = copy.deepcopy(pod) + self.assertEqual(guard_variant(root, pod, "full"), pod["spec"]["initContainers"][0]) + self.assertEqual(pod, original) + + def test_filter_control_preserves_uid_drop_and_legacy_control_preserves_every_rule(self): + root, pod = self.fixture() + filtered = guard_variant(root, pod, "filter-only")["command"][2] + self.assertIn("--uid-owner 1000 -j DROP", filtered) + self.assertNotIn("-t nat", filtered) + legacy = guard_variant(root, pod, "legacy-full")["command"][2] + expected = pod["spec"]["initContainers"][0]["command"][2] + self.assertEqual(legacy.split("; ", 1)[1], expected.replace("iptables ", "iptables-legacy ")) + self.assertIn("|| exit 42", legacy) + + def test_unreviewed_command_image_or_extra_mount_is_never_executed(self): + root, pod = self.fixture() + for key, value in (("image", "other-image"), ("command", ["sh", "-c", "unreviewed"]), + ("volumeMounts", [{"name": "private"}]), ("envFrom", [{"secretRef": {"name": "private"}}])): + changed = copy.deepcopy(pod) + changed["spec"]["initContainers"][0][key] = value + with self.assertRaises(AssertionError): + guard_variant(root, changed, "full") diff --git a/tests/e2e/sre_authority/proxy.py b/tests/e2e/sre_authority/proxy.py new file mode 100644 index 000000000..0a9fdb6d7 --- /dev/null +++ b/tests/e2e/sre_authority/proxy.py @@ -0,0 +1,238 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import importlib.util +import json +import os +import re +import sys +import types + +from .common import AGENT, EPOCH, OPERATORS, PRIVATE, RUNTIME, STANDIN, SYSTEM, require +from .admission import runtime_denials +from .credential_paths import token_secret_denials + +MARKER = "e2e-secret-value-must-not-cross-filter" +SA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount" + + +def install_metrics(h): + # Real metrics-server on the disposable Kind cluster. This is not a fake + # metrics API and does not change any production chart or SRE policy. + if not h.get("apiservice", "v1beta1.metrics.k8s.io"): + h.k("apply", "-f", "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml", + timeout=90) + h.k("patch", "deployment", "metrics-server", "-n", "kube-system", "--type=json", "-p", + json.dumps([{"op": "add", "path": "/spec/template/spec/containers/0/args/-", + "value": "--kubelet-insecure-tls"}])) + h.k("rollout", "status", "deployment/metrics-server", "-n", "kube-system", "--timeout=120s", timeout=130) + def collected(): + response = h.api("GET", "/apis/metrics.k8s.io/v1beta1/nodes") + return response.status_code == 200 and bool(response.json().get("items")) + h.poll("real Kind node metrics", collected, seconds=100, interval=3) + + +def alive_pinned_source(h): + source = h.get("karssandbox", "sre", SYSTEM) + uid = source["metadata"]["uid"] + # An explicit, valid BYO image pin exercises projection preservation. This + # is a sleeping test consumer, not an old binary or a live model/agent run. + h.k("patch", "karssandbox", "sre", "-n", SYSTEM, "--type=merge", "-p", json.dumps({ + "metadata": {"uid": uid, "resourceVersion": source["metadata"]["resourceVersion"]}, + "spec": {"agent": None, "runtime": {"kind": "BYO", "hermes": None, + "byo": {"image": STANDIN, "contractVersion": "v1", "command": ["/bin/sh"], + "args": ["-c", "echo sre-standin-alive; sleep infinity"]}}}})) + def current(): + deployment = h.get("deployment", "sre", RUNTIME) + if not deployment: + return False + template = deployment["spec"]["template"]["spec"] + agent = next((item for item in template["containers"] if item["name"] == "agent"), {}) + runtime = next((entry.get("value") for entry in agent.get("env", []) if entry["name"] == "KARS_RUNTIME_KIND"), None) + return deployment if agent.get("image") == STANDIN and runtime == "BYO" else False + h.poll("registered BYO stand-in projection", current, seconds=150) + h.k("rollout", "status", "deployment/sre", "-n", RUNTIME, "--timeout=150s", timeout=160) + def ready(): + pods = json.loads(h.k("get", "pods", "-n", RUNTIME, "-l", "kars.azure.com/sandbox=sre", "-o", "json"))["items"] + live = [pod for pod in pods if not pod["metadata"].get("deletionTimestamp") + and any(condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in pod.get("status", {}).get("conditions", []))] + return live[0] if len(live) == 1 else False + pod = h.poll("private TLS router and alive stand-in Pod Ready", ready, seconds=100) + require(h.get("karssandbox", "sre", SYSTEM)["metadata"]["uid"] == uid, "Runtime pin replaced the registered source") + return pod + + +def projection_and_files(h, pod): + spec = pod["spec"] + require(spec["serviceAccountName"] == "sandbox", "Projection changed Azure's federated ServiceAccount subject") + require(spec.get("automountServiceAccountToken") is False, "Pod still auto-mounts a Kubernetes JWT") + registration = h.get("karssreregistrations.kars.azure.com", "canonical") + router_sa = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(router_sa and router_sa.get("automountServiceAccountToken") is False + and router_sa["metadata"]["uid"] == registration["status"]["routerServiceAccountUid"], + "Private router identity is not the registered non-automounting ServiceAccount") + agent = next(item for item in spec["containers"] if item["name"] == "agent") + router = next(item for item in spec["containers"] if item["name"] == "inference-router") + require(agent["image"] == STANDIN and router["image"] == "kars-inference-router:e2e", "Selected image pin/artifact changed") + agent_mounts = {mount["name"]: mount["mountPath"] for mount in agent["volumeMounts"]} + router_mounts = {mount["name"]: mount["mountPath"] for mount in router["volumeMounts"]} + require(agent_mounts.get("sre-api-agent") == SA_PATH, "Agent standard paths are not projected from the safe identity") + require("sre-api-private" not in agent_mounts and "router-kubernetes" not in agent_mounts + and "azure-identity-token" not in agent_mounts, "Agent received a private or projected Azure/Kubernetes credential") + require(router_mounts.get("sre-api-private") == "/etc/kars/sre-api" + and router_mounts.get("router-kubernetes") == SA_PATH, "Router lost its private/API credential isolation") + volumes = {volume["name"]: volume for volume in spec["volumes"]} + require(volumes["sre-api-agent"]["secret"]["secretName"] == AGENT, "Agent mapping references wrong Secret") + require(volumes["sre-api-private"]["secret"]["secretName"] == PRIVATE, "Router mapping references wrong Secret") + require({item["key"] for item in volumes["sre-api-agent"]["secret"]["items"]} == {"token", "ca.crt", "namespace"}, + "Agent safe projection exposes unexpected keys") + env = {item["name"]: item.get("value") for item in agent["env"]} + require(env.get("KUBERNETES_SERVICE_HOST") == "127.0.0.1" + and env.get("KUBERNETES_SERVICE_PORT") == "9446", "Legacy HTTPS endpoint is not the local private proxy") + skipped = pod["metadata"].get("annotations", {}).get("azure.workload.identity/skip-containers", "").split(",") + require("agent" in skipped and "egress-guard" in skipped, "Azure token injection is not excluded from the agent") + for name in ("token", "ca.crt", "namespace"): + # Only safe agent files are copied. Private credentials are never + # copied to the host, printed or placed in argv. + value = h.k("exec", "-n", RUNTIME, pod["metadata"]["name"], "-c", "agent", + "--", "cat", f"{SA_PATH}/{name}", user="registrar", timeout=25) + h.write(f"agent/{name}", value) + require((h.work / "agent/namespace").read_text().strip() == RUNTIME, "Standard namespace file changed") + token = (h.work / "agent/token").read_text().strip() + h.token_config("opaque", token) + response = h.api("GET", "/api/v1/namespaces", user="opaque") + require(response.status_code == 401, "Opaque agent credential authenticated to the real Kubernetes API") + review = h.api("POST", "/apis/authentication.k8s.io/v1/tokenreviews", body={ + "apiVersion": "authentication.k8s.io/v1", "kind": "TokenReview", "spec": {"token": token}}, + status=201).json() + require(review["status"].get("authenticated") is not True, "TokenReview accepted the opaque agent credential") + h.passed("Real Pod preserves Sandbox/Azure identity and image pin; only safe standard files reach the agent") + h.passed("Opaque agent credential is rejected by both real API authentication and TokenReview without admin-cert fallback") + + +def load_unchanged_hermes(h, port): + root = h.root / "runtimes/hermes/src/kars_runtime_hermes/plugin" + package = "_kars_e2e_unchanged_sre" + module = types.ModuleType(package) + module.__path__ = [str(root)] + sys.modules[package] = module + loaded = {} + for name in ("sre_kube", "sre"): + spec = importlib.util.spec_from_file_location(f"{package}.{name}", root / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + loaded[name] = module + # Test-only host relocation of the unchanged client's standard file root. + loaded["sre_kube"]._SA_DIR = h.work / "agent" + os.environ["KUBERNETES_SERVICE_HOST"] = "127.0.0.1" + os.environ["KUBERNETES_SERVICE_PORT"] = str(port) + os.environ["NO_PROXY"] = "127.0.0.1,localhost" + return loaded["sre_kube"], loaded["sre"] + + +def assert_filtered(secret): + require(secret.get("kind") == "Secret", "Filtered GET did not return a Secret projection") + require(set(secret.get("data", {})) == {"operator-token", "password"}, "Secret key names were lost") + require(all(value == "" for value in secret["data"].values()), "Secret values crossed the proxy") + require("stringData" not in secret, "stringData crossed the proxy") + for key in ("annotations", "labels", "managedFields"): + require(key not in secret["metadata"], f"Secret {key} copies crossed the proxy") + require(MARKER not in json.dumps(secret), "Secret material survived projection") + + +def secret_list_wire_facts(value, uid): + require(isinstance(value, dict) and value.get("kind") == "SecretList" + and isinstance(value.get("items"), list) and len(value["items"]) == 1 + and isinstance(value["items"][0], dict) and isinstance(value["items"][0].get("metadata"), dict) + and value["items"][0]["metadata"].get("uid") == uid, + "Native wire proof must select only the exact owned synthetic Secret") + item = value["items"][0] + return {"envelopeKind": "SecretList", "itemHasKind": "kind" in item, + "itemHasApiVersion": "apiVersion" in item, "itemMetadataObject": isinstance(item.get("metadata"), dict)} + + +def log_reader_facts(root, value): + from .bootstrap_diagnostics import response_summary + text = value.get("logs") + facts = {"hasError": "error" in value, "hasText": isinstance(text, str), + "standinMarker": isinstance(text, str) and "sre-standin-alive" in text} + error = value.get("error") + status = re.match(r"^([1-5][0-9]{2}) ", error) if isinstance(error, str) else None + if status: + try: + body = json.loads(value.get("body", "")) + except (ValueError, TypeError): + body = None + facts.update(response_summary(int(status[1]), body, root)) + return facts + + +def proxy_acceptance(h): + install_metrics(h) + pod = alive_pinned_source(h) + projection_and_files(h, pod) + runtime_denials(h, pod["metadata"]["name"]) + token_secret_denials(h) + fixture = h.create({"apiVersion": "v1", "kind": "Secret", + "metadata": {"name": f"sre-filter-{h.phase}", "namespace": OPERATORS, + "labels": {"copy": MARKER}, + "annotations": {"kubectl.kubernetes.io/last-applied-configuration": json.dumps({"data": {"copy": MARKER}})}}, + "stringData": {"operator-token": MARKER, "password": MARKER}}) + name = f"sre-filter-{h.phase}" + native = h.api("GET", f"/api/v1/namespaces/{OPERATORS}/secrets?fieldSelector=metadata.name%3D{name}", + status=200).json() + print("SRE-WIRE", json.dumps(secret_list_wire_facts(native, fixture["metadata"]["uid"])), flush=True) + with h.port_forward(pod["metadata"]["name"]) as port: + kube_module, sre = load_unchanged_hermes(h, port) + kube = kube_module.client() + try: + secret = kube.get(f"/api/v1/namespaces/{OPERATORS}/secrets/{name}") + assert_filtered(secret) + listing = kube.get(f"/api/v1/namespaces/{OPERATORS}/secrets", params={"fieldSelector": f"metadata.name={name}"}) + require(listing.get("kind") == "SecretList" and len(listing["items"]) == 1, "Filtered LIST did not reach the real API") + assert_filtered(listing["items"][0]) + h.passed("Unchanged Hermes HTTPS client GET/LIST retains Secret key names but no values or metadata copies") + logs = sre._impl_sre_logs(namespace=RUNTIME, pod=pod["metadata"]["name"], container="agent", tail=20) + print("SRE-LOG-WIRE", json.dumps(log_reader_facts(h.root, logs)), flush=True) + require("error" not in logs and "sre-standin-alive" in logs.get("logs", ""), "Unchanged Hermes raw log reader failed") + metrics = kube.get("/apis/metrics.k8s.io/v1beta1/nodes") + require(metrics.get("kind") == "NodeMetricsList" and metrics.get("items"), "Real metrics did not pass the filtered proxy") + action = sre._create_karssreaction_cr( + action={"type": "RolloutRestart", "namespace": SYSTEM, "kind": "Deployment", "name": "kars-controller"}, + diagnosis="Kind acceptance only", rationale="Pending proposal; never approve or execute", ttl_minutes=5) + created = h.get("karssreaction", action, RUNTIME) + require(created and created["spec"]["approval"] == {"state": "Pending"}, "Real Hermes proposal did not persist Pending") + h.passed("Actual logs, real metrics and unchanged Hermes Pending proposal builder work over Pod TLS; no model run claimed") + raw = kube._ensure_client() + base = f"https://127.0.0.1:{port}" + for path in [ + "/api/v1/namespaces/%6bars-sre/pods", + "/api/v1/namespaces/kars-sre/pods/../secrets", + "/api/v1/namespaces/kars-sre/pods?watch=true", + f"/api/v1/namespaces/kars-sre/pods/{pod['metadata']['name']}/proxy", + f"/api/v1/namespaces/kars-sre/pods/{pod['metadata']['name']}/log?follow=true", + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token", + ]: + # Encode dot segments explicitly so the HTTP client cannot + # normalize an escape into a different legitimate request. + path = path.replace("/../", "/%2e%2e/") + require(raw.get(base + path).status_code == 403, "Unsafe proxy path/query was not rejected") + target = f"/apis/kars.azure.com/v1alpha1/namespaces/{RUNTIME}/karssreactions/{action}" + require(raw.patch(base + target, json={"spec": {"approval": {"state": "Approved"}}}).status_code == 403, + "Self-approval PATCH was not rejected") + require(raw.post(base + f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router/token", + json={"spec": {"audiences": [], "expirationSeconds": 600}}).status_code == 403, + "Token creation escaped the filtered proxy") + require(raw.post(base + f"/api/v1/namespaces/{RUNTIME}/configmaps", + json={"metadata": {"name": "must-not-create"}}).status_code == 403, + "Arbitrary writes escaped the filtered proxy") + invalid = {**created, "metadata": {"name": "self-approved", "namespace": RUNTIME}, + "spec": {**created["spec"], "approval": {"state": "Approved"}}} + invalid.pop("status", None) + require(raw.post(base + f"/apis/kars.azure.com/v1alpha1/namespaces/{RUNTIME}/karssreactions", + json=invalid).status_code == 403, "Approved proposal creation was not rejected") + h.passed("Encoded/proxy/watch/token/write/self-approval paths are rejected, not mistaken for missing API resources") + finally: + kube.close() diff --git a/tests/e2e/sre_authority/registration_schema.py b/tests/e2e/sre_authority/registration_schema.py new file mode 100644 index 000000000..5b5d5ec47 --- /dev/null +++ b/tests/e2e/sre_authority/registration_schema.py @@ -0,0 +1,285 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Real API validation for one public CRD; never a generic error-body logger.""" + +import contextlib +import argparse +import copy +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, build_opener, ProxyHandler + +CONTEXT = "kind-kars-e2e" +CRD_NAME = "karssreregistrations.kars.azure.com" +CRD_PATH = "/apis/apiextensions.k8s.io/v1/customresourcedefinitions" +TEMPLATE = "templates/crd-karssreregistration.yaml" +REPORT_DIR = "e2e-sre-schema-diag" +ORIGINAL_RULE = "self.sandbox.namespace == self.controller.namespace.name" +ESCAPED_RULE = "self.sandbox.__namespace__ == self.controller.__namespace__.name" + + +def require_public_crd(obj): + if (not isinstance(obj, dict) or obj.get("apiVersion") != "apiextensions.k8s.io/v1" + or obj.get("kind") != "CustomResourceDefinition" + or obj.get("metadata", {}).get("name") != CRD_NAME + or obj.get("spec", {}).get("group") != "kars.azure.com" + or obj.get("spec", {}).get("scope") != "Cluster" + or obj.get("spec", {}).get("names", {}).get("kind") != "KarsSRERegistration" + or obj.get("spec", {}).get("names", {}).get("plural") != "karssreregistrations" + or set(obj) - {"apiVersion", "kind", "metadata", "spec"}): + raise AssertionError("Schema diagnostic accepts only the public SRE registration CRD") + + +def public_status(code, body): + """Only this known public schema's Invalid causes may cross the log boundary.""" + report = {"resource": CRD_NAME, "httpStatus": code, "category": "unexpected-response"} + if not isinstance(body, dict): + return report + if code in (200, 201) and body.get("kind") == "CustomResourceDefinition": + if body.get("metadata", {}).get("name") == CRD_NAME: + report["category"] = "accepted" + return report + reasons = {"Invalid", "Forbidden", "Unauthorized", "NotFound", "AlreadyExists", + "Conflict", "BadRequest", "InternalError", "ServiceUnavailable"} + if body.get("kind") == "Status" and body.get("reason") in reasons: + report["category"] = body["reason"] + details = body.get("details", {}) + if (code != 422 or body.get("kind") != "Status" or body.get("reason") != "Invalid" + or not isinstance(details, dict) or details.get("name") != CRD_NAME + or details.get("group") != "apiextensions.k8s.io" + or details.get("kind") != "CustomResourceDefinition"): + return report + causes = [] + supplied = details.get("causes", []) + if not isinstance(supplied, list): + return report + for cause in supplied[:32]: + if not isinstance(cause, dict): + continue + field, message = cause.get("field"), cause.get("message") + if (not isinstance(field, str) or not field.startswith("spec.") + or not isinstance(message, str)): + continue + causes.append({ + "field": re.sub(r"[\x00-\x1f\x7f]", "?", field)[:1024], + "message": re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "?", message)[:16384], + }) + report["causes"] = causes + return report + + +def write_report(root, filename, report): + directory = Path(root) / REPORT_DIR + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + (directory / filename).write_text(json.dumps(report, indent=2) + "\n") + print("SRE-CRD-SCHEMA " + json.dumps(report, sort_keys=True), flush=True) + + +def create_registration_crd(harness, obj): + require_public_crd(obj) + response = harness.api("POST", CRD_PATH, body=obj) + try: + body = response.json() + except (ValueError, TypeError): + body = None + report = public_status(response.status_code, body) + write_report(harness.root, "registration-create.json", report) + if response.status_code != 201 or report["category"] != "accepted": + raise AssertionError( + f"Public SRE registration CRD rejected: HTTP {response.status_code}; " + f"category={report['category']}; see allowlisted schema causes" + ) + return body + + +def command(stage, args, *, root, data=None): + try: + result = subprocess.run(args, cwd=root, input=data, text=True, capture_output=True, + timeout=45, check=False) + except (OSError, subprocess.TimeoutExpired): + raise RuntimeError(f"Public schema preflight {stage} command unavailable/timed out") from None + if result.returncode: + raise RuntimeError(f"Public schema preflight {stage} failed; exit={result.returncode}") from None + return result.stdout + + +def request(port, method, path, obj=None, *, accept="application/json"): + body = None if obj is None else json.dumps(obj).encode() + req = Request(f"http://127.0.0.1:{port}{path}", data=body, method=method, + headers={"Content-Type": "application/json", "Accept": accept}) + opener = build_opener(ProxyHandler({})) + try: + response = opener.open(req, timeout=15) + except HTTPError as error: + response = error + with response: + code = response.code + raw = response.read(1024 * 1024) + try: + return code, json.loads(raw) + except (ValueError, TypeError): + return code, None + + +@contextlib.contextmanager +def kind_proxy(root): + # Read only redacted config to verify the exact disposable context/server. + config = json.loads(command("context", ["kubectl", "--context", CONTEXT, "config", "view", + "--minify", "-o", "json"], root=root)) + contexts, clusters = config.get("contexts", []), config.get("clusters", []) + if (len(contexts) != 1 or contexts[0].get("name") != CONTEXT or len(clusters) != 1 + or urlsplit(clusters[0].get("cluster", {}).get("server", "")).hostname + not in ("localhost", "127.0.0.1", "::1")): + raise RuntimeError("Public schema preflight refuses a non-loopback/non-Kind context") + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + process = subprocess.Popen( + ["kubectl", "--context", CONTEXT, "--request-timeout=15s", "proxy", + "--address=127.0.0.1", f"--port={port}"], + cwd=root, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 25 + version = None + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Public schema preflight proxy exited before readiness") + try: + code, observed = request(port, "GET", "/version") + if code == 200 and isinstance(observed, dict) and isinstance(observed.get("gitVersion"), str): + version = {key: observed.get(key) for key in ("major", "minor", "gitVersion")} + break + except (URLError, TimeoutError): + pass + time.sleep(0.2) + if version is None: + raise RuntimeError("Public schema preflight proxy readiness timed out") + yield port, version + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def escaped_namespace_candidate(obj): + """Diagnostic-only candidate: preserve the constraint and its wire schema.""" + candidate = copy.deepcopy(obj) + rules = candidate["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["x-kubernetes-validations"] + matches = [rule for rule in rules if rule.get("rule") == ORIGINAL_RULE] + if len(matches) != 1: + raise RuntimeError("Expected public namespace equality rule is absent; candidate probe refused") + matches[0]["rule"] = ESCAPED_RULE + return candidate + + +def exercise_instances(root, port, obj, method, path, accepted, prefix): + # This helper is used only in the isolated schema CI job. The full harness + # keeps its preflight dry-run-only so historical fixture setup is unchanged. + code, body = request(port, method, path, obj) + if code != accepted or public_status(code, body)["category"] != "accepted": + write_report(root, f"{prefix}-install.json", public_status(code, body)) + raise RuntimeError("Public schema could not be installed in the disposable schema cluster") + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + code, current = request(port, "GET", f"{CRD_PATH}/{CRD_NAME}") + if code == 200 and any(condition.get("type") == "Established" and condition.get("status") == "True" + for condition in current.get("status", {}).get("conditions", [])): + break + time.sleep(0.5) + else: + raise RuntimeError("Public registration CRD did not become Established") + instance = { + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, + "spec": { + "controller": {"namespace": {"name": "kars-system", "uid": "fixture-system"}, + "deployment": {"name": "kars-controller", "uid": "fixture-controller"}, "release": "kars"}, + "sandbox": {"namespace": "kars-system", "name": "sre", "uid": "fixture-source"}, + "runtimeNamespace": {"name": "kars-sre", "uid": "fixture-runtime"}, + }, + } + results = [] + for label, expected, message in [ + ("canonical-matching-namespace", 201, None), + ("foreign-source-namespace", 422, "SRE must be registered in its controller/release namespace"), + ("noncanonical-name", 422, "The canonical SRE registration is the only supported instance"), + ]: + probe = copy.deepcopy(instance) + if label == "foreign-source-namespace": + probe["spec"]["sandbox"]["namespace"] = "other-system" + elif label == "noncanonical-name": + probe["metadata"]["name"] = "other" + code, response = request(port, "POST", "/apis/kars.azure.com/v1alpha1/karssreregistrations?dryRun=All", probe) + matched = code == expected + if message: + matched = matched and isinstance(response, dict) and response.get("reason") == "Invalid" and any( + message in cause.get("message", "") for cause in response.get("details", {}).get("causes", []) + ) + else: + matched = matched and isinstance(response, dict) and response.get("kind") == "KarsSRERegistration" + results.append({"case": label, "httpStatus": code, "expectedStatus": expected, "matched": bool(matched)}) + write_report(root, f"{prefix}-instances.json", {"cases": results}) + if not matched: + raise RuntimeError("Public registration instance did not satisfy the exact expected schema invariant") + + +def preflight(root, *, candidate=False, exercise=False): + rendered = command("render", ["helm", "template", "kars", str(root / "deploy/helm/kars"), + "--namespace", "kars-system", "--show-only", TEMPLATE], root=root) + # Existing kubectl parses YAML; strict client validation remains enabled. + obj = json.loads(command("conversion", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=rendered)) + require_public_crd(obj) + if candidate: + obj = escaped_namespace_candidate(obj) + prefix = "namespace-accessor-candidate" if candidate else "validation" + with kind_proxy(root) as (port, version): + write_report(root, "versions.json", {"apiServer": version, "context": CONTEXT}) + code, existing = request(port, "GET", f"{CRD_PATH}/{CRD_NAME}") + if code == 404: + method, path, accepted = "POST", CRD_PATH, 201 + elif code == 200 and public_status(code, existing)["category"] == "accepted": + method, path, accepted = "PUT", f"{CRD_PATH}/{CRD_NAME}", 200 + for key in ("uid", "resourceVersion"): + obj["metadata"][key] = existing["metadata"][key] + else: + raise RuntimeError(f"Public schema preflight could not inspect CRD; HTTP {code}") + code, body = request(port, method, path + "?dryRun=All", obj) + report = public_status(code, body) + write_report(root, f"{prefix}.json", report) + if code != accepted or report["category"] != "accepted": + raise RuntimeError(f"Public SRE registration schema rejected; HTTP {code}") + if exercise: + exercise_instances(root, port, obj, method, path, accepted, prefix) + + +if __name__ == "__main__": + os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--namespace-accessor-candidate", action="store_true", + help="Probe only the API-evidenced field-accessor candidate; does not edit production files") + parser.add_argument("--exercise", action="store_true", help="Exercise public instance invariants in the isolated schema job") + args = parser.parse_args() + try: + preflight(Path(__file__).resolve().parents[3], + candidate=args.namespace_accessor_candidate, exercise=args.exercise) + except Exception as error: + # Deliberately do not expose arbitrary exception text, command output, + # argv, config material, or an unrelated HTTP response. + print(f"SRE-CRD-SCHEMA-FAIL category={type(error).__name__}", flush=True) + raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/registration_schema_test.py b/tests/e2e/sre_authority/registration_schema_test.py new file mode 100644 index 000000000..a623f2216 --- /dev/null +++ b/tests/e2e/sre_authority/registration_schema_test.py @@ -0,0 +1,186 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure checks for the public-CRD diagnostic boundary, not real API evidence.""" + +import contextlib +import copy +import json +from pathlib import Path +import types +import unittest +from unittest.mock import patch + +from sre_authority.common import command_error_category +from sre_authority import registration_schema as schema + +PRIVATE = "DO-NOT-LOG-PRIVATE-AUTH-OR-SECRET-BODY" + + +def crd(): + return { + "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", + "metadata": {"name": schema.CRD_NAME}, + "spec": {"group": "kars.azure.com", "scope": "Cluster", + "names": {"kind": "KarsSRERegistration", "plural": "karssreregistrations"}}, + } + + +def invalid(): + return { + "kind": "Status", "status": "Failure", "reason": "Invalid", "code": 422, + "details": { + "name": schema.CRD_NAME, "group": "apiextensions.k8s.io", "kind": "CustomResourceDefinition", + "causes": [{"field": "spec.versions[0].schema.openAPIV3Schema.properties[spec].x-kubernetes-validations[0].rule", + "message": "public schema compile diagnostic\n ^"}], + }, + } + + +class RegistrationSchemaTests(unittest.TestCase): + def test_native_kubectl_invalid_classification_does_not_echo_body(self): + message = f'The CustomResourceDefinition "{schema.CRD_NAME}" is invalid: {PRIVATE}' + self.assertEqual(command_error_category(message), "Invalid") + self.assertNotIn(PRIVATE, command_error_category(message)) + self.assertEqual(command_error_category(f"Error from server (Forbidden): {PRIVATE}"), "Forbidden") + + def test_only_exact_public_schema_422_causes_are_exposed(self): + report = schema.public_status(422, invalid()) + self.assertEqual(report["category"], "Invalid") + self.assertEqual(len(report["causes"]), 1) + self.assertIn("public schema compile diagnostic", report["causes"][0]["message"]) + for code, kind, name, group in [ + (403, "CustomResourceDefinition", schema.CRD_NAME, "apiextensions.k8s.io"), + (500, "CustomResourceDefinition", schema.CRD_NAME, "apiextensions.k8s.io"), + (422, "Secret", "private-secret", ""), + (422, "CustomResourceDefinition", "other.example.test", "apiextensions.k8s.io"), + (422, "CustomResourceDefinition", schema.CRD_NAME, "other.group"), + ]: + body = invalid() + body["message"] = PRIVATE + body["details"].update({"kind": kind, "name": name, "group": group}) + body["details"]["causes"][0]["message"] = PRIVATE + with self.subTest(code=code, kind=kind): + self.assertNotIn(PRIVATE, json.dumps(schema.public_status(code, body))) + body = invalid() + body["details"]["causes"] = [{"field": "data.token", "message": PRIVATE}] + self.assertNotIn(PRIVATE, json.dumps(schema.public_status(422, body))) + + def test_malformed_responses_do_not_become_success_or_echo_raw_content(self): + for body in (None, [], PRIVATE, {"message": PRIVATE}, + {**invalid(), "details": None}, + {**invalid(), "details": {**invalid()["details"], "causes": None}}): + with self.subTest(body=type(body).__name__): + report = schema.public_status(422, body) + self.assertNotEqual(report["category"], "accepted") + self.assertNotIn(PRIVATE, json.dumps(report)) + + def test_general_secret_commands_cannot_use_the_public_crd_diagnostic(self): + for obj in ( + {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": schema.CRD_NAME}}, + {**crd(), "data": {"token": PRIVATE}}, + {**crd(), "metadata": {"name": "other.example.test"}}, + ): + with self.assertRaises(AssertionError): + schema.require_public_crd(obj) + + def test_actual_create_path_surfaces_only_allowlisted_api_evidence(self): + response = types.SimpleNamespace(status_code=422, json=invalid) + harness = types.SimpleNamespace(root=Path("."), api=lambda *_args, **_kwargs: response) + with patch.object(schema, "write_report") as write: + with self.assertRaisesRegex(AssertionError, "HTTP 422"): + schema.create_registration_crd(harness, crd()) + self.assertEqual(write.call_args.args[1], "registration-create.json") + self.assertEqual(write.call_args.args[2]["causes"], schema.public_status(422, invalid())["causes"]) + + def test_fast_preflight_uses_real_server_dry_run_and_rejects_invalid_schema(self): + root = Path(__file__).resolve().parents[3] + seen = [] + def request(_port, method, path, body=None): + seen.append((method, path, body)) + return (404, {}) if method == "GET" else (422, invalid()) + def command(stage, _args, **_kwargs): + return "public yaml" if stage == "render" else json.dumps(crd()) + with patch.object(schema, "kind_proxy", return_value=contextlib.nullcontext((1, {"gitVersion": "v1.31.0"}))), \ + patch.object(schema, "request", side_effect=request), \ + patch.object(schema, "command", side_effect=command), \ + patch.object(schema, "write_report") as write: + with self.assertRaisesRegex(RuntimeError, "HTTP 422"): + schema.preflight(root) + self.assertEqual(seen[1][:2], ("POST", schema.CRD_PATH + "?dryRun=All")) + self.assertEqual(write.call_args.args[2]["category"], "Invalid") + + def test_reused_kind_cluster_still_validates_update_instead_of_accepting_already_exists(self): + root = Path(__file__).resolve().parents[3] + existing = crd() + existing["metadata"].update({"uid": "crd-uid", "resourceVersion": "42"}) + seen = [] + def request(_port, method, path, body=None): + seen.append((method, path, copy.deepcopy(body))) + return (200, existing) + with patch.object(schema, "kind_proxy", return_value=contextlib.nullcontext((1, {"gitVersion": "v1.31.0"}))), \ + patch.object(schema, "request", side_effect=request), \ + patch.object(schema, "command", side_effect=lambda stage, *_args, **_kwargs: + "public yaml" if stage == "render" else json.dumps(crd())), \ + patch.object(schema, "write_report"): + schema.preflight(root) + self.assertEqual(seen[1][:2], ("PUT", schema.CRD_PATH + "/" + schema.CRD_NAME + "?dryRun=All")) + self.assertEqual(seen[1][2]["metadata"]["uid"], "crd-uid") + self.assertEqual(seen[1][2]["metadata"]["resourceVersion"], "42") + + def test_candidate_changes_only_field_accessors_not_constraint_or_wire_properties(self): + original = crd() + original["spec"]["versions"] = [{"schema": {"openAPIV3Schema": { + "x-kubernetes-validations": [{"rule": "self.metadata.name == 'canonical'"}], + "properties": {"spec": { + "properties": {"sandbox": {"properties": {"namespace": {"type": "string"}}}}, + "x-kubernetes-validations": [{"rule": schema.ORIGINAL_RULE, "message": "unchanged"}], + }}, + }}}] + expected = copy.deepcopy(original) + expected["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"][ + "x-kubernetes-validations"][0]["rule"] = schema.ESCAPED_RULE + candidate = schema.escaped_namespace_candidate(original) + self.assertEqual(candidate, expected) + self.assertIn(schema.ORIGINAL_RULE, json.dumps(original)) + + def test_candidate_instance_checks_require_both_original_invariants(self): + seen = [] + def request(_port, method, path, body=None): + seen.append((method, path, body)) + if path == schema.CRD_PATH: + return 201, crd() + if method == "GET": + return 200, {**crd(), "status": {"conditions": [{"type": "Established", "status": "True"}]}} + message = None + if body["metadata"]["name"] != "canonical": + message = "The canonical SRE registration is the only supported instance" + elif body["spec"]["sandbox"]["namespace"] != body["spec"]["controller"]["namespace"]["name"]: + message = "SRE must be registered in its controller/release namespace" + if message: + return 422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [{"message": message}]}} + return 201, body + with patch.object(schema, "request", side_effect=request), patch.object(schema, "write_report") as write: + schema.exercise_instances(Path("."), 1, crd(), "POST", schema.CRD_PATH, 201, "candidate") + self.assertEqual(len(write.call_args.args[2]["cases"]), 3) + self.assertTrue(all(case["matched"] for case in write.call_args.args[2]["cases"])) + self.assertTrue(all("?dryRun=All" in path for method, path, _ in seen if "karssreregistrations?" in path)) + + def test_fast_ci_gate_uses_the_harness_pins_without_waiting_for_rust_images(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + job = workflow.split(" sre-crd-schema:\n", 1)[1].split(" helm-lint:\n", 1)[0] + for required in ("version: v0.24.0", "version: v1.30.5", "tests/e2e/kind-config.yaml", + "registration_schema.py", "e2e-sre-schema-diag/validation.json"): + self.assertIn(required, job) + for forbidden in ("needs:", "cargo ", "docker/build-push", "--validate=false", "continue-on-error"): + self.assertNotIn(forbidden, job) + self.assertIn("--namespace-accessor-candidate --exercise", job) + self.assertIn("if: failure()", job) + runner = (root / "tests/e2e/run.sh").read_text() + self.assertLess(runner.index('python3 "$SCRIPT_DIR/sre_authority/registration_schema.py"'), + runner.index("\n build_images\n")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/replication_controller_test.py b/tests/e2e/sre_authority/replication_controller_test.py new file mode 100644 index 000000000..91eaeb98f --- /dev/null +++ b/tests/e2e/sre_authority/replication_controller_test.py @@ -0,0 +1,188 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Unit checks for the nonexecuting native RC admission probe orchestration.""" + +import copy +import unittest +from types import SimpleNamespace +from unittest.mock import Mock + +from sre_authority.admission import rc_update_preview, replication_controller_cases +from sre_authority.common import PRIVATE, RUNTIME + + +class Response: + def __init__(self, code, body): + self.status_code = code + self.body = body + + def json(self): + return copy.deepcopy(self.body) + + +class Harness: + def __init__(self, *, allow_private=False, wrong_denial=False, conflict=False, replace=False, + update_conflict=False): + self.calls = [] + self.object = None + self.completed = [] + self.allow_private = allow_private + self.wrong_denial = wrong_denial + self.conflict = conflict + self.replace = replace + self.update_conflict = update_conflict + + def passed(self, message): + self.completed.append(message) + + def poll(self, _message, operation, **_options): + if not operation(): + raise AssertionError("fixture polling did not converge") + + def api(self, method, path, *, body=None, user="admin", status=None): + self.calls.append((method, path, copy.deepcopy(body), user)) + if "selfsubjectaccessreviews" in path: + attributes = body["spec"]["resourceAttributes"] + allowed = attributes["resource"] == "replicationcontrollers" or attributes.get("subresource") == "log" + response = Response(201, {"status": {"allowed": allowed}}) + elif method == "GET": + if self.object is not None and self.replace: + self.object["metadata"]["uid"] = "replacement" + response = Response(200, self.object) if self.object is not None else Response(404, {}) + elif method == "DELETE": + assert body["preconditions"] == { + "uid": self.object["metadata"]["uid"], + "resourceVersion": self.object["metadata"]["resourceVersion"], + } + if self.conflict: + self.conflict = False + self.object["metadata"]["resourceVersion"] = "2" + response = Response(409, {"reason": "Conflict"}) + else: + self.object = None + response = Response(200, {}) + elif "?dryRun=All" in path: + if method == "PUT" and self.update_conflict: + self.update_conflict = False + self.object["metadata"]["resourceVersion"] = "2" + return Response(409, {"kind": "Status", "reason": "Conflict"}) + if "template" not in body["spec"]: + response = Response(422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [ + {"field": "spec.template", "reason": "FieldValueRequired"}, + ]}}) + if status is not None: + assert response.status_code in (status if isinstance(status, tuple) else (status,)) + return response + pod = body["spec"].get("template", {}).get("spec", {}) + private = bool(pod.get("volumes")) or pod.get("serviceAccountName") == "sre-api-router" + containers = pod.get("containers", []) + pod.get("initContainers", []) + private = private or any(container.get("envFrom") or any( + item.get("valueFrom", {}).get("secretKeyRef", {}).get("name") == PRIVATE + for item in container.get("env", [])) for container in containers) + if private and not self.allow_private: + policy = "unrelated" if self.wrong_denial else "kars-sre-private-workloads" + response = Response(403, {"kind": "Status", "reason": "Forbidden", + "message": f"{policy} denied request"}) + else: + response = Response(201 if method == "POST" else 200, body) + else: + assert method == "POST" and body["spec"]["replicas"] == 0 + assert body["spec"]["template"]["spec"].get("volumes") is None + self.object = copy.deepcopy(body) + self.object["metadata"].update(uid="owned-rc", resourceVersion="1") + response = Response(201, self.object) + if status is not None: + assert response.status_code in (status if isinstance(status, tuple) else (status,)) + return response + + +class ReplicationControllerProofTests(unittest.TestCase): + def test_all_private_forms_are_denied_for_create_and_update_without_workload_execution(self): + harness = Harness() + replication_controller_cases(harness) + self.assertEqual(len(harness.completed), 1) + self.assertIsNone(harness.object) + attempts = [call for call in harness.calls if call[0] in ("POST", "PUT") + and "replicationcontrollers" in call[1] and "?dryRun=All" in call[1]] + self.assertEqual(len(attempts), 15) + self.assertEqual(sum(method == "PUT" for method, *_ in attempts), 7) + for _method, _path, body, user in attempts: + self.assertEqual(user, "tenant") + self.assertEqual(body["spec"]["replicas"], 0) + if "template" in body["spec"]: + pod = body["spec"]["template"]["spec"] + self.assertEqual(pod["schedulerName"], "kars-e2e-admission-never-schedule") + self.assertEqual(pod["containers"][0]["command"], ["/bin/true"]) + self.assertEqual(pod["containers"][0]["imagePullPolicy"], "Never") + self.assertFalse(any("/secrets/" in path or "/pods/" in path for _, path, *_ in harness.calls)) + rights = [body["spec"]["resourceAttributes"] for _, path, body, _ in harness.calls + if "selfsubjectaccessreviews" in path] + self.assertIn({"group": "", "resource": "pods", "verb": "get", "namespace": RUNTIME, + "subresource": "log"}, rights) + self.assertIn({"group": "", "resource": "pods", "verb": "create", "namespace": ""}, rights) + self.assertTrue(any(PRIVATE in str(body) for _, _, body, _ in attempts)) + + def test_an_admitted_private_controller_or_wrong_denial_never_counts_as_success(self): + for options in ({"allow_private": True}, {"wrong_denial": True}): + harness = Harness(**options) + with self.assertRaises(AssertionError): + replication_controller_cases(harness) + self.assertFalse(harness.completed) + self.assertIsNone(harness.object) + + def test_cleanup_conflict_uses_a_fresh_version_without_dropping_uid_fence(self): + harness = Harness(conflict=True) + replication_controller_cases(harness) + deletions = [body for method, _, body, _ in harness.calls if method == "DELETE"] + self.assertEqual([body["preconditions"] for body in deletions], [ + {"uid": "owned-rc", "resourceVersion": "1"}, + {"uid": "owned-rc", "resourceVersion": "2"}, + ]) + + def test_update_status_conflict_retries_with_the_same_uid_and_fresh_version(self): + harness = Harness(update_conflict=True) + replication_controller_cases(harness) + updates = [body for method, _, body, _ in harness.calls if method == "PUT"] + self.assertEqual(updates[0]["metadata"]["uid"], "owned-rc") + self.assertEqual(updates[1]["metadata"]["uid"], "owned-rc") + self.assertEqual(updates[0]["metadata"]["resourceVersion"], "1") + self.assertEqual(updates[1]["metadata"]["resourceVersion"], "2") + self.assertEqual(len(harness.completed), 1) + + def test_update_preview_never_retries_identity_or_content_changes(self): + original = {"metadata": {"name": "fixture", "uid": "owned", "resourceVersion": "1"}, + "spec": {"replicas": 0}} + for change in ["uid", "spec", "deletionTimestamp"]: + refreshed = copy.deepcopy(original) + refreshed["metadata"]["resourceVersion"] = "2" + if change == "uid": + refreshed["metadata"]["uid"] = "foreign" + elif change == "spec": + refreshed["spec"]["replicas"] = 1 + else: + refreshed["metadata"]["deletionTimestamp"] = "2026-09-10T12:00:00Z" + api = Mock(side_effect=[Response(200, original), Response(409, {"reason": "Conflict"}), + Response(200, refreshed)]) + with self.assertRaises(AssertionError): + rc_update_preview(SimpleNamespace(api=api), "/fixture", original, original) + self.assertEqual(api.call_count, 3) + + def test_update_preview_returns_other_denials_without_retry(self): + original = {"metadata": {"uid": "owned", "resourceVersion": "1"}, "spec": {}} + for code in [403, 422, 500]: + denial = Response(code, {"reason": "Forbidden"}) + api = Mock(side_effect=[Response(200, original), denial]) + self.assertIs(rc_update_preview(SimpleNamespace(api=api), "/fixture", original, original), denial) + self.assertEqual(api.call_count, 2) + + def test_replacement_refuses_update_and_cleanup_of_the_foreign_object(self): + harness = Harness(replace=True) + with self.assertRaises(AssertionError): + replication_controller_cases(harness) + self.assertFalse(any(method in ("PUT", "DELETE") for method, *_ in harness.calls)) + self.assertFalse(harness.completed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/smoke_ordering_test.py b/tests/e2e/sre_authority/smoke_ordering_test.py new file mode 100644 index 000000000..5ff018731 --- /dev/null +++ b/tests/e2e/sre_authority/smoke_ordering_test.py @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Smoke assertions wait for their own asynchronous controller resources.""" + +from pathlib import Path +import re +import subprocess +import unittest + + +class SmokeOrderingTests(unittest.TestCase): + def test_resource_checks_use_bounded_exact_waits_and_still_fail_when_absent(self): + source = (Path(__file__).resolve().parents[1] / "run.sh").read_text() + for function, kind, name in [ + ("test_networkpolicy_created", "networkpolicy", "sandbox-policy"), + ("test_serviceaccount_created", "serviceaccount", "sandbox"), + ]: + match = re.search(rf"^{function}\(\) \{{\n.*?^\}}", source, re.MULTILINE | re.DOTALL) + self.assertIsNotNone(match) + for available in [True, False]: + with self.subTest(function=function, available=available): + script = f""" +wait_for_resource() {{ + printf 'WAIT %s %s %s %s\\n' "$1" "$2" "$3" "$4" + return {0 if available else 1} +}} +pass() {{ printf 'PASS %s\\n' "$1"; }} +fail() {{ printf 'FAIL %s\\n' "$1"; return 1; }} +{match.group(0)} +{function} +""" + result = subprocess.run(["bash", "-c", script], text=True, capture_output=True, + timeout=5, check=False) + self.assertEqual(result.returncode, 0 if available else 1) + self.assertIn(f"WAIT {kind} {name} kars-e2e-test 30\n", result.stdout) + self.assertIn("PASS " if available else "FAIL ", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/staging_compatibility_test.py b/tests/e2e/sre_authority/staging_compatibility_test.py new file mode 100644 index 000000000..33b220e7e --- /dev/null +++ b/tests/e2e/sre_authority/staging_compatibility_test.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure contracts for live-controller staging; no Kubernetes execution.""" + +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from sre_authority.migration import running_controller_stage + + +def fixture(): + objects = { + "deployment": {"spec": {"replicas": 1}, "status": {"availableReplicas": 1}}, + "karssandbox": {"metadata": {"uid": "source"}, "status": { + "conditions": [{"type": "Ready", "status": "False"}]}}, + "clusterrolebinding": {"metadata": {"uid": "reader"}, "subjects": [{"name": "sandbox"}]}, + } + h = SimpleNamespace( + get=lambda kind, *_args: deepcopy(objects.get(kind)), + cli=Mock(), passed=Mock(), + poll=lambda _label, predicate, **_kwargs: predicate(), + ) + return h, objects + + +class StagingCompatibilityTests(unittest.TestCase): + def test_real_stage_runs_before_enrollment_without_changing_the_ready_gate(self): + h, _objects = fixture() + with patch("sre_authority.migration.policies_ready") as ready: + running_controller_stage(h) + h.cli.assert_called_once_with("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", timeout=180) + ready.assert_called_once_with(h) + h.passed.assert_called_once() + source = (Path(__file__).parent / "migration.py").read_text() + migration = source.split("def legacy_migration(h):", 1)[1].split("def rollback_guard", 1)[0] + self.assertLess(migration.index("running_controller_stage(h)"), migration.index('"enroll"')) + self.assertIn('h.cli("authority", "migrate"', migration) + self.assertIn("h.wait_ready()", migration) + + def test_zero_or_unavailable_controller_cannot_mask_the_helm_watcher_deadlock(self): + for key in ("replicas", "availableReplicas"): + h, objects = fixture() + objects["deployment"]["spec" if key == "replicas" else "status"][key] = 0 + with self.subTest(key=key), self.assertRaises(AssertionError): + running_controller_stage(h) + h.cli.assert_not_called() + h.passed.assert_not_called() + + def test_existing_enrollment_is_not_a_valid_pre_enrollment_fixture(self): + h, objects = fixture() + objects["karssreregistrations.kars.azure.com"] = {"metadata": {"uid": "registration"}} + with self.assertRaises(AssertionError): + running_controller_stage(h) + h.cli.assert_not_called() + + def test_stage_timeout_and_policy_typechecking_failure_remain_fatal(self): + h, _objects = fixture() + h.cli.side_effect = RuntimeError("stage timeout") + with self.assertRaisesRegex(RuntimeError, "stage timeout"): + running_controller_stage(h) + h.passed.assert_not_called() + h.cli.side_effect = None + with patch("sre_authority.migration.policies_ready", side_effect=RuntimeError("policy typechecking")): + with self.assertRaisesRegex(RuntimeError, "policy typechecking"): + running_controller_stage(h) + h.passed.assert_not_called() + + def test_native_fixture_checks_exact_historical_action_spec_and_uid_after_stage(self): + source = (Path(__file__).parent / "fixtures.py").read_text() + self.assertIn('action_params.pop("additionalProperties", None) is True', source) + self.assertIn('action_params["x-kubernetes-preserve-unknown-fields"] = True', source) + self.assertIn('action_after["metadata"]["uid"] == action_before["metadata"]["uid"]', source) + self.assertIn('action_after["spec"] == action_spec', source) + self.assertLess(source.index("action_before ="), source.index('h.cli("authority", "stage"')) + self.assertGreater(source.index("action_after ="), source.index('h.cli("authority", "stage"')) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/headlamp-plugin/package-lock.json b/tools/headlamp-plugin/package-lock.json index f4e0e5616..442fcb529 100644 --- a/tools/headlamp-plugin/package-lock.json +++ b/tools/headlamp-plugin/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "devDependencies": { "@kinvolk/headlamp-plugin": "^0.13.0", - "vite": "^6.4.3" + "vite": "6.4.3" } }, "node_modules/@adobe/css-tools": { @@ -11457,9 +11457,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { diff --git a/tools/headlamp-plugin/package.json b/tools/headlamp-plugin/package.json index cc15e06d0..01965b62e 100644 --- a/tools/headlamp-plugin/package.json +++ b/tools/headlamp-plugin/package.json @@ -20,6 +20,7 @@ }, "overrides": { "vitest": "^4.1.8", - "tmp": "^0.2.6" + "tmp": "^0.2.6", + "js-yaml": "4.3.2" } }