From 5543c401463f6df68436e4d0902b063f6ac5507d Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 10:32:47 +0200 Subject: [PATCH 01/62] feat(sre): require registered authority and isolate Kubernetes credentials Introduce operator-controlled UID enrollment, reviewed legacy-grant migration, renewable router-private Kubernetes identity and a filtered HTTPS compatibility proxy. Preserve Azure identity and pinned Hermes interfaces; block legacy token aliases and Secret-watch authority. Add lifecycle preflights, retained admission gates and real Kind acceptance coverage. Live API qualification and genuine sign-offs remain pending. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 20 +- Cargo.lock | 27 + Cargo.toml | 4 + ci/no-custom-crypto.sh | 1 + ci/no-stubs.sh | 1 + ci/security-audit-required.sh | 2 +- cli/src/commands/destroy.ts | 6 + cli/src/commands/dev/local-k8s.ts | 5 + cli/src/commands/push-apply.ts | 2 + cli/src/commands/sre-authority.ts | 82 +++ cli/src/commands/sre.test.ts | 62 +- cli/src/commands/sre.ts | 71 +- cli/src/commands/up.ts | 9 +- cli/src/commands/up/fast_upgrade.ts | 2 + cli/src/commands/upgrade.ts | 3 + cli/src/lib/sre-authority.test.ts | 249 +++++++ cli/src/lib/sre-authority.ts | 259 ++++++++ cli/src/lib/sre-source.ts | 79 +++ cli/src/lib/sre-stage.ts | 104 +++ .../testing/shared-security-guards.test.ts | 33 + cli/src/testing/sre-authority.test.ts | 75 +++ controller/Cargo.toml | 2 + controller/src/main.rs | 11 + controller/src/providers/mod.rs | 1 + controller/src/providers/signing.rs | 7 + controller/src/providers/sre_tls.rs | 51 ++ controller/src/reconciler/mod.rs | 116 ++-- controller/src/reconciler/pod_spec.rs | 110 +--- controller/src/sre_authority.rs | 189 ++++++ controller/src/sre_authority/admission.rs | 70 ++ controller/src/sre_authority/bindings.rs | 623 ++++++++++++++++++ .../src/sre_authority/credential_guard.rs | 32 + controller/src/sre_authority/credentials.rs | 423 ++++++++++++ controller/src/sre_authority/live.rs | 166 +++++ controller/src/sre_authority/migration.rs | 353 ++++++++++ controller/src/sre_authority/pod.rs | 158 +++++ controller/src/sre_authority/privacy_tests.rs | 424 ++++++++++++ controller/src/sre_authority/tests.rs | 564 ++++++++++++++++ controller/src/sre_registration.rs | 191 ++++++ .../templates/crd-karssreregistration.yaml | 115 ++++ .../templates/sre-authority-admission.yaml | 329 +++++++++ .../templates/sre-authority-consumers.yaml | 221 +++++++ .../kars/templates/sre-authority-rbac.yaml | 138 ++++ deploy/helm/kars/templates/sre.yaml | 98 +-- docs/how-to/sre-authority.md | 212 ++++++ .../2026-09-08-sre-authority-prerequisite.md | 145 ++++ inference-router/Cargo.toml | 3 + inference-router/src/lib.rs | 3 + inference-router/src/main.rs | 8 + inference-router/src/sre_proxy/backend.rs | 365 ++++++++++ inference-router/src/sre_proxy/mod.rs | 320 +++++++++ inference-router/src/sre_proxy/policy.rs | 409 ++++++++++++ inference-router/src/sre_proxy/tests.rs | 380 +++++++++++ shared/sre_privacy.rs | 82 +++ tests/e2e/namespace-ownership.sh | 60 +- tests/e2e/run.sh | 27 +- tests/e2e/sre-authority.py | 65 ++ tests/e2e/sre-authority.sh | 31 + tests/e2e/sre_authority/__init__.py | 4 + tests/e2e/sre_authority/admission.py | 122 ++++ tests/e2e/sre_authority/common.py | 348 ++++++++++ tests/e2e/sre_authority/credential_paths.py | 223 +++++++ tests/e2e/sre_authority/fixtures.py | 226 +++++++ tests/e2e/sre_authority/harness_test.py | 207 ++++++ tests/e2e/sre_authority/migration.py | 252 +++++++ tests/e2e/sre_authority/proxy.py | 206 ++++++ 66 files changed, 8881 insertions(+), 305 deletions(-) create mode 100644 cli/src/commands/sre-authority.ts create mode 100644 cli/src/lib/sre-authority.test.ts create mode 100644 cli/src/lib/sre-authority.ts create mode 100644 cli/src/lib/sre-source.ts create mode 100644 cli/src/lib/sre-stage.ts create mode 100644 cli/src/testing/shared-security-guards.test.ts create mode 100644 cli/src/testing/sre-authority.test.ts create mode 100644 controller/src/providers/sre_tls.rs create mode 100644 controller/src/sre_authority.rs create mode 100644 controller/src/sre_authority/admission.rs create mode 100644 controller/src/sre_authority/bindings.rs create mode 100644 controller/src/sre_authority/credential_guard.rs create mode 100644 controller/src/sre_authority/credentials.rs create mode 100644 controller/src/sre_authority/live.rs create mode 100644 controller/src/sre_authority/migration.rs create mode 100644 controller/src/sre_authority/pod.rs create mode 100644 controller/src/sre_authority/privacy_tests.rs create mode 100644 controller/src/sre_authority/tests.rs create mode 100644 controller/src/sre_registration.rs create mode 100644 deploy/helm/kars/templates/crd-karssreregistration.yaml create mode 100644 deploy/helm/kars/templates/sre-authority-admission.yaml create mode 100644 deploy/helm/kars/templates/sre-authority-consumers.yaml create mode 100644 deploy/helm/kars/templates/sre-authority-rbac.yaml create mode 100644 docs/how-to/sre-authority.md create mode 100644 docs/security-audits/2026-09-08-sre-authority-prerequisite.md create mode 100644 inference-router/src/sre_proxy/backend.rs create mode 100644 inference-router/src/sre_proxy/mod.rs create mode 100644 inference-router/src/sre_proxy/policy.rs create mode 100644 inference-router/src/sre_proxy/tests.rs create mode 100644 shared/sre_privacy.rs create mode 100644 tests/e2e/sre-authority.py create mode 100644 tests/e2e/sre-authority.sh create mode 100644 tests/e2e/sre_authority/__init__.py create mode 100644 tests/e2e/sre_authority/admission.py create mode 100644 tests/e2e/sre_authority/common.py create mode 100644 tests/e2e/sre_authority/credential_paths.py create mode 100644 tests/e2e/sre_authority/fixtures.py create mode 100644 tests/e2e/sre_authority/harness_test.py create mode 100644 tests/e2e/sre_authority/migration.py create mode 100644 tests/e2e/sre_authority/proxy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 716b086df..c05451762 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,9 @@ jobs: # transitive oci-client→reqwest) compiles natively with the # runner's gcc. run: cargo build --release --workspace + - name: Legacy Hermes HTTPS client test dependency + if: needs.changes.outputs.code == 'true' + run: python3 -m pip install 'httpx==0.28.1' - name: cargo nextest run (release) if: needs.changes.outputs.code == 'true' # Reuses the target/release/ artefacts the previous step just @@ -572,7 +575,7 @@ jobs: # Fetch enough history to diff. git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true if git diff --name-only "$base" "$head" 2>/dev/null \ - | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then + | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|shared/|runtimes/hermes/src/kars_runtime_hermes/plugin/sre|cli/src/(commands/sre|lib/sre|lib/namespace-ownership)|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" @@ -616,6 +619,21 @@ jobs: docker system prune -af --volumes >/dev/null 2>&1 || true df -h + - name: Set up Node for real SRE authority CLI acceptance + if: steps.paths.outputs.run == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + cache: npm + cache-dependency-path: cli/package-lock.json + - name: Build locked CLI for Kind acceptance + if: steps.paths.outputs.run == 'true' + working-directory: cli + run: npm ci && npm run build + - name: SRE Kind unchanged Hermes HTTPS client dependency + if: steps.paths.outputs.run == 'true' + run: python3 -m pip install 'httpx==0.28.1' + # Pre-built binaries from build-rust are COPY'd into the # distroless runtime images; no `cargo build` runs inside Docker. # Result: each image build is ~30s instead of ~8min. diff --git a/Cargo.lock b/Cargo.lock index 44e7cfddf..c9a7577b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2501,6 +2501,7 @@ dependencies = [ "oci-client 0.16.1", "prometheus", "rand 0.9.4", + "rcgen", "regex", "reqwest 0.12.28", "rustls", @@ -2511,6 +2512,7 @@ dependencies = [ "sha2 0.10.9", "sigstore", "thiserror 2.0.18", + "time", "tokio", "tokio-tungstenite 0.28.0", "tracing", @@ -2552,8 +2554,10 @@ dependencies = [ "prometheus", "proptest", "rand 0.9.4", + "rcgen", "reqwest 0.12.28", "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_yaml", @@ -2562,6 +2566,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls", "tokio-stream", "tokio-tungstenite 0.26.2", "tokio-util", @@ -4034,6 +4039,19 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -6343,6 +6361,15 @@ dependencies = [ "tls_codec", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 6e451a538..5e3ef964b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,10 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" # with "Could not automatically determine the process-level # CryptoProvider" when multiple feature flags resolve. rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs"] } +rcgen = { version = "0.13.2", default-features = false, features = ["aws_lc_rs", "pem"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs"] } +rustls-pemfile = "2" +time = "0.3" tower = { version = "0.5", features = ["limit"] } tower-http = { version = "0.6", features = ["trace", "cors"] } hyper = "1" diff --git a/ci/no-custom-crypto.sh b/ci/no-custom-crypto.sh index 8e282611c..e9f1f8c13 100755 --- a/ci/no-custom-crypto.sh +++ b/ci/no-custom-crypto.sh @@ -67,6 +67,7 @@ ALLOW_PATHS=( # Production paths to scan. PROD_PATHS=( + 'shared/' 'controller/src/' 'inference-router/src/' 'cli/src/' diff --git a/ci/no-stubs.sh b/ci/no-stubs.sh index ae5d8beb4..12839ca3d 100755 --- a/ci/no-stubs.sh +++ b/ci/no-stubs.sh @@ -19,6 +19,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" PROD_PATHS=( + 'shared/' 'controller/src/' 'inference-router/src/' 'cli/src/' diff --git a/ci/security-audit-required.sh b/ci/security-audit-required.sh index b9c37bc25..e2aa54480 100755 --- a/ci/security-audit-required.sh +++ b/ci/security-audit-required.sh @@ -17,7 +17,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" # Capability-introducing paths — mirrors §4.4 of the plan. -CAP_RE='^(controller/src/(crd|reconcilers|admission)|inference-router/src/(mcp|a2a|providers|routes)|cli/src/(commands|migrate|adapters)|runtimes/openclaw/src/(core|index\.ts)|sandbox-images/[^/]+/(Dockerfile|entrypoint\.sh)|cli/profiles/|deploy/seccomp/|deploy/helm/kars/files/)' +CAP_RE='^(controller/src/(crd|reconcilers|admission)|inference-router/src/(mcp|a2a|providers|routes)|cli/src/(commands|migrate|adapters)|runtimes/openclaw/src/(core|index\.ts)|sandbox-images/[^/]+/(Dockerfile|entrypoint\.sh)|cli/profiles/|deploy/seccomp/|deploy/helm/kars/files/|shared/.*\.rs$)' changed=$(git diff --name-only "${BASE_REF}...HEAD" 2>/dev/null || git diff --name-only HEAD) # Exclude test files — they exercise capabilities but don't introduce diff --git a/cli/src/commands/destroy.ts b/cli/src/commands/destroy.ts index 41897be4c..40ffd1822 100644 --- a/cli/src/commands/destroy.ts +++ b/cli/src/commands/destroy.ts @@ -4,6 +4,7 @@ import { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; +import { assertDestroySafe } from "../lib/sre-authority.js"; export function destroyCommand(): Command { const cmd = new Command("destroy"); @@ -22,6 +23,11 @@ export function destroyCommand(): Command { const rg = options.resourceGroup || `kars-${options.region}`; // Propagate --context to every kubectl invocation in this command. const kctlCtx = options.context ? ["--context", options.context] : []; + if ((!options.local || options.cloud) && (!name || name === "sre" || options.all)) { + const { execa } = await import("execa"); + await assertDestroySafe((file,args,commandOptions) => + execa(file,[...kctlCtx,...args],commandOptions)); + } if (options.all) { // Full teardown — delete the entire resource group diff --git a/cli/src/commands/dev/local-k8s.ts b/cli/src/commands/dev/local-k8s.ts index c3aadd388..32f7f921b 100644 --- a/cli/src/commands/dev/local-k8s.ts +++ b/cli/src/commands/dev/local-k8s.ts @@ -31,6 +31,7 @@ import { ensureAgtRepo, ensureAgtWheels } from "../../lib/agt-bootstrap.js"; import { resolveBundledAsset, requireBundledAsset, findRepoRootOrNull } from "../../lib/repo-assets.js"; import { buildCopilotFallbackChain } from "../../github-copilot.js"; import { CLAIM, prepareCredentialNamespace } from "../../lib/namespace-ownership.js"; +import { assertSafeMutation } from "../../lib/sre-authority.js"; export interface LocalK8sOptions { /** Sandbox / agent name. Reused as Helm release name suffix. */ @@ -1337,6 +1338,10 @@ export async function runLocalK8s(opts: LocalK8sOptions): Promise { stepper.step(`Ensuring kind cluster '${opts.clusterName}' exists…`); await ensureCluster(tools.kind, opts.clusterName, tools.env); + await assertSafeMutation((file,args,commandOptions) => execa( + file === "kubectl" ? tools.kubectl : file, + ["--context", `kind-${opts.clusterName}`, ...args], commandOptions, + )); stepper.done(`kind cluster '${opts.clusterName}' is ready`); // Ensure the three local-dev images exist AND match the host arch. diff --git a/cli/src/commands/push-apply.ts b/cli/src/commands/push-apply.ts index ff47f1405..baf0c142e 100644 --- a/cli/src/commands/push-apply.ts +++ b/cli/src/commands/push-apply.ts @@ -11,6 +11,7 @@ import { import { coreImageValues, imageValueArgs, PUSH_COMPONENTS, resolvePushedArtifacts, type PushedImage } from "../lib/image-targets.js"; import { inspectCoreInstallation, recheckCoreOwnership, requireHealthyDeployment, updateLegacyCore, verifyCoreConfiguration } from "../lib/core-image-apply.js"; import { inspectSandboxPlans, refreshSandboxImages } from "../lib/sandbox-image-apply.js"; +import { assertSafeMutation } from "../lib/sre-authority.js"; export interface PushApplyResult { applied: string[]; @@ -31,6 +32,7 @@ export async function applyPushedImages( const buildOnly = images.filter(item => item.name === "sandbox-base").map(item => item.name); const deployable = images.filter(item => item.name !== "sandbox-base"); if (!deployable.length) throw new Error("sandbox-base is build-only; no deployment was applied"); + await assertSafeMutation(execute); const isMesh = (item: PushedImage) => item.name === "relay" || item.name === "registry"; const selectedCore = deployable.filter(item => !isMesh(item)); const selectedMesh = deployable.some(isMesh); diff --git a/cli/src/commands/sre-authority.ts b/cli/src/commands/sre-authority.ts new file mode 100644 index 000000000..5609fb5b1 --- /dev/null +++ b/cli/src/commands/sre-authority.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Command } from "commander"; +import { execa } from "execa"; +import { requireBundledAsset } from "../lib/repo-assets.js"; +import { enroll, preview, requireRegistrar, retire, waitForAuthority, type Execute } from "../lib/sre-authority.js"; +import { stageSource } from "../lib/sre-source.js"; +import { stageAuthority } from "../lib/sre-stage.js"; + +function executor(context?: string): Execute { + return (file, args, options) => execa(file, [ + ...(context ? [file === "helm" ? "--kube-context" : "--context", context] : []), ...args, + ], options); +} + +export function authorityCommand(): Command { + const command = new Command("authority").description("Stage, review, enroll and retire cluster-authorized SRE privacy"); + const common = (name: string) => command.command(name) + .option("--namespace ", "Controller/release namespace", "kars-system") + .option("--release ", "Owning Helm release", "kars") + .option("--context ", "Kubernetes context"); + common("stage") + .description("Explicit registrar staging: install authority APIs/controller while retaining legacy grants unchanged") + .requiredOption("--controller-image ", "Qualified prerequisite controller repository:tag") + .requiredOption("--router-image ", "Qualified prerequisite router repository:tag") + .option("--dry-run", "Server-side preview without deployment changes") + .action(async options => { + const execute = executor(options.context); + await stageAuthority(execute,requireBundledAsset("deploy/helm/kars"),options.namespace,options.release, + options.controllerImage,options.routerImage,!!options.dryRun); + console.log("Authority controller staged. Preview and explicitly enroll the exact SRE source/grants before normal upgrades."); + }); + common("preview").description("Read exact enrollment identities and legacy grants; no mutations") + .action(async options => { + const spec = await preview(executor(options.context),options.namespace,options.release); + console.log(JSON.stringify(spec,null,2)); + for (const binding of spec.legacyBindings) { + console.log(`--binding '${binding.kind}/${binding.namespace ?? ""}/${binding.name}=${binding.uid}@${binding.resourceVersion}'`); + } + if (spec.legacyConsumer) console.log(`--consumer '${spec.legacyConsumer.uid}@${spec.legacyConsumer.resourceVersion}'`); + }); + common("stage-source").description("Atomically create a genuinely new, unprivileged SRE source and wait for its exact claim") + .option("--model ", "SRE model deployment") + .action(async options => { + const execute=executor(options.context); + await requireRegistrar(execute); + const result=await execute("helm",["template",options.release,requireBundledAsset("deploy/helm/kars"), + "--namespace",options.namespace,"--show-only","templates/sre.yaml", + "--set","sre.enabled=true","--set","azure.workloadIdentity.clientId=dummy", + ...(options.model?["--set-string",`sre.model=${options.model}`]:[])],{stdio:"pipe"}); + const created=await stageSource(execute,result.stdout,options.namespace,options.release); + console.log(`Created and claimed source: --sandbox-uid ${created.uid} --namespace-uid ${created.namespaceUid}`); + }); + common("enroll").description("Enroll only reviewed source/namespace and grant UIDs under cluster registrar authority") + .requiredOption("--sandbox-uid ", "Reviewed canonical Sandbox UID") + .requiredOption("--namespace-uid ", "Reviewed claimed runtime namespace UID") + .option("--binding ", "Exact binding review from preview; repeat per binding", (value: string, all: string[]) => [...all,value], []) + .option("--consumer ", "Reviewed legacy SRE Deployment") + .option("--registration-uid ", "Required when updating an existing registration") + .option("--resource-version ", "Required when updating an existing registration") + .option("--dry-run", "Print the enrollment without writing it") + .action(async options => { + const execute = executor(options.context); + const spec = await preview(execute,options.namespace,options.release); + console.log(await enroll(execute,spec,options,!!options.dryRun)); + }); + common("migrate").description("Wait for the controller to complete the explicitly enrolled migration") + .action(async options => { + await requireRegistrar(executor(options.context)); + await waitForAuthority(executor(options.context),"Ready"); + console.log("SRE authority Ready: legacy credentials denied and private renewable identity configured."); + }); + common("retire").description("Disable and retire a reviewed registration before SRE uninstall") + .requiredOption("--registration-uid ", "Reviewed registration UID") + .requiredOption("--resource-version ", "Reviewed registration resourceVersion") + .action(async options => { + await retire(executor(options.context),options.registrationUid,options.resourceVersion); + console.log("SRE authority Retired; owned private grants and credentials are revoked."); + }); + return command; +} diff --git a/cli/src/commands/sre.test.ts b/cli/src/commands/sre.test.ts index 436c98fa5..1a7f98f48 100644 --- a/cli/src/commands/sre.test.ts +++ b/cli/src/commands/sre.test.ts @@ -13,10 +13,45 @@ vi.mock("../lib/repo-assets.js", () => ({ requireBundledAsset: () => "/test/char const releases = JSON.stringify([{ name: "kars", namespace: "kars-system" }]); const controller = JSON.stringify({ apiVersion: "apps/v1", kind: "Deployment", - metadata: { name: "kars-controller", namespace: "kars-system", uid: "controller-uid" }, + metadata: { name: "kars-controller", namespace: "kars-system", uid: "controller-uid", resourceVersion: "1" }, }); +function authority(args: readonly string[]): { stdout: string } { + const metadata = (name: string, uid = name) => ({ name, uid, resourceVersion: "1", generation: 1 }); + let value: unknown; + if (args.includes("can-i")) return { stdout: "yes" }; + if (args.includes("crd")) value = { metadata: metadata("karssreregistrations.kars.azure.com") }; + if (args.includes("karssreregistrations.kars.azure.com")) value = { + metadata: metadata("canonical"), + spec: { + enabled: true, + controller: { namespace: { name: "kars-system", uid: "kars-system" }, + deployment: { name: "kars-controller", uid: "controller-uid" }, release: currentRelease }, + sandbox: { namespace: "kars-system", name: "sre", uid: "source" }, + runtimeNamespace: { name: "kars-sre", uid: "kars-sre" }, + }, + status: { phase: "Ready", observedGeneration: 1, privacyRevision: "kars.azure.com/sre-privacy/v2" }, + }; + if (args.includes("namespace")) value = args.includes("kars-sre") + ? { metadata: { ...metadata("kars-sre"), annotations: { + "kars.azure.com/namespace-claim-version": "v1", + "kars.azure.com/sandbox-namespace": "kars-system", + "kars.azure.com/sandbox-name": "sre", + "kars.azure.com/sandbox-uid": "source", + } } } + : { metadata: metadata("kars-system") }; + if (args.includes("karssandbox")) value = { + metadata: { ...metadata("sre", "source"), namespace: "kars-system", + annotations: { "kars.azure.com/namespace-uid": "kars-sre" } }, + }; + if (args.includes("clusterrolebindings") || args.includes("rolebindings")) value = { items: [] }; + return { stdout: value ? JSON.stringify(value) : "" }; +} + +let currentRelease = "kars"; + beforeEach(() => { + currentRelease = "kars"; execute.mockReset(); vi.spyOn(console, "log").mockImplementation(() => {}); }); @@ -34,7 +69,7 @@ describe("SRE controller upgrade namespace preflight", () => { expect(args.slice(0, 2)).toEqual(["--context", "test-context"]); return { stdout: '{"items":[]}' }; } - return { stdout: "" }; + return authority(args); }); await sreCommand().parseAsync(["node", "sre", "install", "--no-wait", "--context", "test-context"]); const calls = execute.mock.calls; @@ -89,14 +124,22 @@ describe("SRE controller upgrade namespace preflight", () => { }); it("permits fresh installation only after successful inventory and explicit controller absence", async () => { - execute.mockImplementation(async (file, args) => ({ - stdout: file === "helm" && args[0] === "list" ? "[]" : "", - })); + execute.mockImplementation(async (file, args) => { + if (file === "helm" && args[0] === "list") return { stdout: "[]" }; + if (args[0] === "-n") return { stdout: "" }; + if (args.includes("deployment") && args.includes("kars-controller")) return { stdout: controller }; + return authority(args); + }); await sreCommand().parseAsync(["node", "sre", "install", "--no-wait"]); - expect(execute.mock.calls.map(([file, args]) => [file, args[0]])).toEqual([ - ["helm", "list"], ["kubectl", "-n"], ["helm", "install"], + expect(execute.mock.calls.slice(0, 2).map(([file, args]) => [file, args[0]])).toEqual([ + ["helm", "list"], ["kubectl", "-n"], ]); expect(execute.mock.calls[1][1]).toContain("--ignore-not-found"); + const installs = execute.mock.calls.filter(([file, args]) => file === "helm" && args[0] === "install"); + expect(installs).toHaveLength(1); + expect(installs[0][1]).toContain("sre.enabled=false"); + expect(execute.mock.calls.some(([, args]) => args.includes("--take-ownership") || args.includes("--force-conflicts"))).toBe(false); + expect(execute.mock.calls.some(([file, args]) => file === "helm" && args[0] === "upgrade" && args.includes("sre.enabled=true"))).toBe(true); }); it.each([ @@ -105,13 +148,14 @@ describe("SRE controller upgrade namespace preflight", () => { { target: "kars", names: ["other.release"] }, { target: "a".repeat(53), names: ["a".repeat(53)] }, ])("accepts Helm-compatible release inventory for $target", async ({ target, names }) => { + currentRelease = target; execute.mockImplementation(async (file, args) => { if (file === "helm" && args[0] === "list") { return { stdout: JSON.stringify(names.map(name => ({ name, namespace: "kars-system" }))) }; } - if (file === "kubectl" && args.includes("deployment")) return { stdout: controller }; + if (file === "kubectl" && args.includes("deployment") && args.includes("kars-controller")) return { stdout: controller }; if (file === "kubectl" && args.includes("karssandboxes")) return { stdout: '{"items":[]}' }; - return { stdout: "" }; + return authority(args); }); await sreCommand().parseAsync(["node", "sre", "install", "--no-wait", "--release", target]); const mode = names.includes(target) ? "upgrade" : "template"; diff --git a/cli/src/commands/sre.ts b/cli/src/commands/sre.ts index 09e8e71e6..6ecc47d80 100644 --- a/cli/src/commands/sre.ts +++ b/cli/src/commands/sre.ts @@ -6,6 +6,9 @@ import chalk from "chalk"; import { execa } from "execa"; import { requireBundledAsset } from "../lib/repo-assets.js"; import { inspectNamespaceOwnership } from "../lib/namespace-ownership.js"; +import { authorityCommand } from "./sre-authority.js"; +import { assertDestroySafe, assertSafeMutation, enroll, get, preview, registration, requireRegistrar, waitForAuthority } from "../lib/sre-authority.js"; +import { stageSource } from "../lib/sre-source.js"; const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; @@ -27,6 +30,7 @@ const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z export function sreCommand(): Command { const cmd = new Command("sre"); cmd.description("Manage the built-in kars-sre agent (Kubernetes SRE on the cluster)"); + cmd.addCommand(authorityCommand()); cmd .command("install") @@ -74,11 +78,9 @@ export function sreCommand(): Command { // B. operator deployed via `kars dev --target local-k8s` // (which renders `helm template | kubectl apply` and so // never creates a helm release record) → use `helm template - // | kubectl apply --server-side --force-conflicts` with - // `sre.enabled=true` baked in. The chart is already in - // the cluster; this just adds the SRE bits idempotently. - // C. no chart at all → `helm install` with --take-ownership + - // a fallback workload-identity client-id (local dev). + // | kubectl apply --server-side` without force adoption. + // C. no chart at all → install unprivileged core first, then + // atomically create and enroll the fresh SRE source. let mode: "upgrade" | "template" | "install" = "install"; const listArgs = ["list", "-n", options.namespace, "--all", "-o", "json"]; if (options.context) listArgs.push("--kube-context", options.context); @@ -124,6 +126,43 @@ export function sreCommand(): Command { execa(file, [...(options.context ? ["--context", options.context] : []), ...args], commandOptions)); } + const execute = (file: string, args: readonly string[], commandOptions: { stdio: "pipe"; input?: string }) => + execa(file, [...(options.context ? [file === "helm" ? "--kube-context" : "--context", options.context] : []), ...args], commandOptions); + await requireRegistrar(execute); + if (mode === "install") { + await execute("helm", ["install", options.release, chartPath, "--namespace", options.namespace, + "--create-namespace", "--set", "sre.enabled=false", + "--set", "azure.workloadIdentity.clientId=dummy", "--wait", "--timeout", "8m"], { stdio: "pipe" }); + mode = "upgrade"; + } + if (!await get(execute, "crd", "karssreregistrations.kars.azure.com")) { + throw new Error("Stage the SRE authority prerequisite controller/APIs before installing SRE"); + } + const enrolled = await registration(execute); + if (enrolled) { + if (enrolled.spec?.controller?.namespace?.name !== options.namespace + || enrolled.spec?.controller?.release !== options.release) { + throw new Error("Canonical SRE is registered to another source/release; no privilege was granted"); + } + if (enrolled.spec?.enabled === false) { + throw new Error("Retired SRE authority must be explicitly re-enrolled with reviewed current UIDs before install"); + } + await assertSafeMutation(execute); + } else { + const rendered = await execute("helm", ["template", options.release, chartPath, + "--namespace", options.namespace, "--show-only", "templates/sre.yaml", + "--set", "sre.enabled=true", "--set", "azure.workloadIdentity.clientId=dummy", + ...(options.model ? ["--set-string", `sre.model=${options.model}`] : [])], { stdio: "pipe" }); + const created = await stageSource(execute, rendered.stdout, options.namespace, options.release); + const spec = await preview(execute, options.namespace, options.release); + if (spec.legacyBindings.length) throw new Error("A legacy grant appeared during staging; explicit review is required"); + if (spec.legacyConsumer) throw new Error("An SRE consumer appeared during staging; explicitly review its UID/resourceVersion before enrollment"); + await enroll(execute, spec, { + sandboxUid: created.uid, namespaceUid: created.namespaceUid, binding: [], + }, false); + await waitForAuthority(execute, "Ready"); + } + const helmArgs = mode === "upgrade" ? [ @@ -137,14 +176,8 @@ export function sreCommand(): Command { // release values predate fields like runtimes.hermes — a plain // --reuse-values would carry the gap forward and fail templating. "--reset-then-reuse-values", - // --force-conflicts: helm 4 uses server-side apply by default, - // which conflicts with field managers from prior `kubectl set - // image` / `kars push --apply` runs that touched the same - // fields. This flag tells SSA to take ownership on conflict, - // matching the operator's intent (helm-managed chart is the - // source of truth). - "--force-conflicts", "--set", "sre.enabled=true", + "--set", "sre.authorityStage=false", ] : mode === "template" ? [ @@ -165,15 +198,6 @@ export function sreCommand(): Command { chartPath, "--namespace", options.namespace, "--create-namespace", - "--force-conflicts", - // --take-ownership: adopt resources that already exist in the - // cluster but don't carry helm metadata (the kars-system - // namespace, default-deny NetworkPolicy, etc. created - // out-of-band by a prior `kars dev` or partial helm - // install). Without this, install dies on the first such - // resource with a "cannot be imported" error. Requires - // helm >= 3.17 (`kars dev` pins helm 4 — safe). - "--take-ownership", "--set", "sre.enabled=true", // Brand-new chart install on a fresh cluster has no prior // azure.workloadIdentity.clientId — use a dummy fallback for @@ -205,7 +229,6 @@ export function sreCommand(): Command { "apply", "-f", "-", "--server-side", - "--force-conflicts", ], { input: stdout, @@ -280,9 +303,11 @@ export function sreCommand(): Command { chartPath, "--namespace", options.namespace, "--reset-then-reuse-values", - "--force-conflicts", "--set", "sre.enabled=false", ]; + await assertDestroySafe((file,args,commandOptions) => execa(file,[ + ...(options.context ? [file === "helm" ? "--kube-context" : "--context",options.context] : []),...args, + ],commandOptions)); if (options.context) helmArgs.push("--kube-context", options.context); console.log(chalk.cyan("▸ disabling kars-sre via helm upgrade --reuse-values…")); diff --git a/cli/src/commands/up.ts b/cli/src/commands/up.ts index c958dd2a3..708cf91a9 100644 --- a/cli/src/commands/up.ts +++ b/cli/src/commands/up.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { Command } from "commander"; +import { assertSafeMutation } from "../lib/sre-authority.js"; import chalk from "chalk"; import { existsSync } from "fs"; import * as path from "path"; @@ -605,6 +606,7 @@ Auto-resume: ], { stdio: "pipe" }); stepper.done("kubectl configured"); markPhaseDone("kubectl", {}, resumeTopology); + await assertSafeMutation(execa); // ── Step 6: Get images into ACR ────────────────────────────── const acr = acrLoginServer.replace(".azurecr.io", ""); @@ -782,11 +784,8 @@ Auto-resume: // small system nodes. 10m avoids a spurious "context deadline // exceeded" while k8s is still legitimately rolling out. "--timeout", "10m", - // Take ownership of fields previously written by `kubectl apply` - // or `kubectl patch` (e.g. CRDs / ClusterRoles touched out-of-band - // during prior debugging). Without this, Helm's server-side apply - // refuses with "conflict with kubectl-client-side-apply" and the - // whole `kars up` flow fails after the 18-min image build. + // Preserve the existing core field-manager behavior; the SRE + // preflight above rejects unreviewed grant migration before this. "--force-conflicts", ]; if (foundryEndpoint) { diff --git a/cli/src/commands/up/fast_upgrade.ts b/cli/src/commands/up/fast_upgrade.ts index a65e67440..daa621f3a 100644 --- a/cli/src/commands/up/fast_upgrade.ts +++ b/cli/src/commands/up/fast_upgrade.ts @@ -14,6 +14,7 @@ import { requireBundledAsset } from "../../lib/repo-assets.js"; import { cliReleaseTag } from "../../lib/version.js"; import { rolloutRestartAll } from "../upgrade.js"; import { inspectNamespaceOwnership } from "../../lib/namespace-ownership.js"; +import { assertSafeMutation } from "../../lib/sre-authority.js"; export interface UpOptionsForUpgrade { upgrade?: boolean; @@ -38,6 +39,7 @@ export async function runFastUpgrade(options: UpOptionsForUpgrade): Promise={ + "namespace//kars-system":{kind:"Namespace",metadata:{name:"kars-system",uid:"control-ns",resourceVersion:"1"}}, + "deployment/kars-system/kars-controller":{kind:"Deployment",metadata:{name:"kars-controller",namespace:"kars-system",uid:"controller",resourceVersion:"1"}}, + "karssandbox/kars-system/sre":{kind:"KarsSandbox",metadata:{name:"sre",namespace:"kars-system",uid:"source",resourceVersion:"1", + annotations:{"kars.azure.com/namespace-uid":"runtime-ns"}}}, + "namespace//kars-sre":{kind:"Namespace",metadata:{name:"kars-sre",uid:"runtime-ns",resourceVersion:"1", + annotations:{"kars.azure.com/namespace-claim-version":"v1","kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre","kars.azure.com/sandbox-uid":"source"}}}, + "crd//karssreregistrations.kars.azure.com":{kind:"CustomResourceDefinition",metadata:{name:"karssreregistrations.kars.azure.com",uid:"crd",resourceVersion:"1"}}, + }; + const bindings:any[]=[]; + let use=true; + const execute=vi.fn(async (_file,args,options)=>{ + if(args[0]==="auth")return {stdout:use?"yes":"no"}; + if(args[0]==="get"&&(args[1]==="clusterrolebindings"||args[1]==="rolebindings")) + return {stdout:JSON.stringify({items:args[1]==="clusterrolebindings"?bindings:[]})}; + if(args[0]==="get"){ + const namespace=args.includes("-n")?args[args.indexOf("-n")+1]:""; + const object=objects[`${args[1]}/${namespace}/${args[2]}`]; + return {stdout:object?JSON.stringify(object):""}; + } + if(args[0]==="create"){ + const body=JSON.parse(options.input!); + objects["karssreregistrations.kars.azure.com//canonical"]={...body,metadata:{...body.metadata,uid:"registration",resourceVersion:"1",generation:1}}; + return {stdout:JSON.stringify(objects["karssreregistrations.kars.azure.com//canonical"])}; + } + return {stdout:""}; + }); + return {objects,bindings,execute,denyUse:()=>{use=false}}; +} + +describe("SRE cluster registrar boundary",()=>{ + it("previews only metadata and never enrolls based on labels or a foreign namespace",async()=>{ + const f=fixture(); + const result=await preview(f.execute,"kars-system","kars"); + expect(result.sandbox.uid).toBe("source"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + f.objects["namespace//kars-sre"].metadata.annotations["kars.azure.com/sandbox-namespace"]="foreign"; + await expect(preview(f.execute,"kars-system","kars")).rejects.toThrow(/different|foreign|claim/i); + }); + + it("requires explicit registrar permission and reviewed UIDs",async()=>{ + const f=fixture(); + const spec=await preview(f.execute,"kars-system","kars"); + f.denyUse(); + await expect(enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:[]},false)).rejects.toThrow("registrar"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="create"||args[0]==="patch")).toBe(false); + }); + + it("requires every legacy binding and exact resource version before enrollment",async()=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"legacy",uid:"binding",resourceVersion:"4"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"kars-sre-reader"}, + subjects:[{kind:"ServiceAccount",name:"sandbox",namespace:"kars-sre"}]}); + const spec=await preview(f.execute,"kars-system","kars"); + await expect(enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:[]},false)).rejects.toThrow("Every legacy binding"); + await enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:["ClusterRoleBinding//legacy=binding@4"]},false); + const body=JSON.parse(f.execute.mock.calls.find(([,args])=>args[0]==="create")![2].input!); + expect(body.kind).toBe("KarsSRERegistration"); + expect(body.metadata.namespace).toBeUndefined(); + }); + + it("blocks ordinary mutation and rollback while old grants or authority remain",async()=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"legacy",uid:"binding",resourceVersion:"1"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"kars-sre-reader"}, + subjects:[{kind:"ServiceAccount",name:"sandbox",namespace:"kars-sre"}]}); + await expect(assertSafeMutation(f.execute)).rejects.toThrow("Legacy SRE grants"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + f.bindings.length=0; + f.objects["karssreregistrations.kars.azure.com//canonical"]={metadata:{name:"canonical",uid:"registration",resourceVersion:"1",generation:1}, + spec:{enabled:true},status:{phase:"Ready",observedGeneration:1}}; + await expect(assertRollbackSafe(f.execute)).rejects.toThrow("Rollback"); + await expect(assertDestroySafe(f.execute)).rejects.toThrow("Retire"); + }); + + it("dry-run enrollment performs no mutation",async()=>{ + const f=fixture(); + const spec=await preview(f.execute,"kars-system","kars"); + const result=await enroll(f.execute,spec,{sandboxUid:"source",namespaceUid:"runtime-ns",binding:[]},true); + expect(JSON.parse(result).spec.sandbox.uid).toBe("source"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="create"||args[0]==="patch")).toBe(false); + }); + + it.each([false,true])("replaces the full reviewed re-enrollment spec (consumer present: %s)",async hasConsumer=>{ + const f=fixture(); + if(hasConsumer) f.objects["deployment/kars-sre/sre"]={kind:"Deployment", + metadata:{name:"sre",namespace:"kars-sre",uid:"new-consumer",resourceVersion:"9"}}; + const spec=await preview(f.execute,"kars-system","kars"); + const key="karssreregistrations.kars.azure.com//canonical"; + f.objects[key]={metadata:{name:"canonical",uid:"registration",resourceVersion:"7",generation:2}, + spec:{...spec,enabled:false,legacyConsumer:{namespace:"kars-sre",name:"sre",uid:"retired-consumer",resourceVersion:"2"}, + legacyBindings:[{name:"retired-grant"}]},status:{phase:"Retired",observedGeneration:2}}; + const execute=vi.fn(async(file,args,options)=>{ + if(args[0]!=="patch") return f.execute(file,args,options); + expect(args).toContain("--type=json"); + const operations=JSON.parse(args[args.indexOf("-p")+1]); + expect(operations).toEqual([ + {op:"test",path:"/metadata/uid",value:"registration"}, + {op:"test",path:"/metadata/resourceVersion",value:"7"}, + {op:"replace",path:"/spec",value:spec}, + ]); + f.objects[key].spec=structuredClone(operations[2].value); + return {stdout:""}; + }); + await enroll(execute,spec,{ + sandboxUid:"source",namespaceUid:"runtime-ns",binding:[], + registrationUid:"registration",resourceVersion:"7", + ...(hasConsumer?{consumer:"new-consumer@9"}:{}), + },false); + expect(f.objects[key].spec).toEqual(spec); + expect(f.objects[key].spec.legacyConsumer).toEqual(hasConsumer?spec.legacyConsumer:undefined); + expect(f.objects[key].metadata.uid).toBe("registration"); + }); + + it("does not patch re-enrollment against a stale registration identity",async()=>{ + const f=fixture(); + const spec=await preview(f.execute,"kars-system","kars"); + f.objects["karssreregistrations.kars.azure.com//canonical"]={ + metadata:{name:"canonical",uid:"replacement-registration",resourceVersion:"8",generation:1},spec}; + await expect(enroll(f.execute,spec,{ + sandboxUid:"source",namespaceUid:"runtime-ns",binding:[], + registrationUid:"registration",resourceVersion:"7", + },false)).rejects.toThrow("reviewed registration UID/resourceVersion"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="patch")).toBe(false); + }); + + it("refuses to stage over any preexisting source or namespace",async()=>{ + const f=fixture(); + await expect(stageSource(f.execute,"","kars-system","kars")).rejects.toThrow("already exists"); + delete f.objects["karssandbox/kars-system/sre"]; + await expect(stageSource(f.execute,"","kars-system","kars")).rejects.toThrow("Existing kars-sre"); + expect(f.execute.mock.calls.some(([,args])=>args[0]==="create"||args[0]==="patch")).toBe(false); + }); + + it("captures CREATE UID and never converts a racing 409 into adoption",async()=>{ + const f=fixture(); + delete f.objects["karssandbox/kars-system/sre"]; + delete f.objects["namespace//kars-sre"]; + const base=f.execute; + const execute=vi.fn(async(file,args,options)=>{ + if(args[0]==="create")throw new Error("409 AlreadyExists"); + return base(file,args,options); + }); + const rendered=JSON.stringify({apiVersion:"kars.azure.com/v1alpha1",kind:"KarsSandbox", + metadata:{name:"sre",namespace:"kars-system"},spec:{runtime:{kind:"Hermes"}}}); + await expect(stageSource(execute,rendered,"kars-system","kars")).rejects.toThrow("409"); + const createIndex=execute.mock.calls.findIndex(([,args])=>args[0]==="create"); + expect(createIndex).toBeGreaterThanOrEqual(0); + expect(execute.mock.calls.slice(createIndex+1)).toEqual([]); + }); + + it("enrolls the actual newly created source only after its namespace claim converges",async()=>{ + const f=fixture(); + const source=f.objects["karssandbox/kars-system/sre"]; + const runtime=f.objects["namespace//kars-sre"]; + delete f.objects["karssandbox/kars-system/sre"]; + delete f.objects["namespace//kars-sre"]; + const execute=vi.fn(async(file,args,options)=>{ + if(args[0]==="create"&&JSON.parse(options.input!).kind==="KarsSandbox") { + f.objects["karssandbox/kars-system/sre"]=source; + f.objects["namespace//kars-sre"]=runtime; + return {stdout:JSON.stringify(source)}; + } + return f.execute(file,args,options); + }); + const created=await stageSource(execute,JSON.stringify(source),"kars-system","kars"); + expect(created).toEqual({uid:"source",namespaceUid:"runtime-ns"}); + await enroll(execute,await preview(execute,"kars-system","kars"),{ + sandboxUid:created.uid,namespaceUid:created.namespaceUid,binding:[], + },false); + expect(f.objects["karssreregistrations.kars.azure.com//canonical"].spec.sandbox.uid).toBe("source"); + expect(execute.mock.calls.filter(([,args])=>args[0]==="create")).toHaveLength(2); + }); + + it("waits through controller migration without treating transient drain as failure",async()=>{ + vi.useFakeTimers(); + try { + const f=fixture(); + const key="karssreregistrations.kars.azure.com//canonical"; + f.objects[key]={metadata:{name:"canonical",uid:"registration",resourceVersion:"1",generation:1}, + status:{phase:"Migrating",observedGeneration:1,privacyRevision:"kars.azure.com/sre-privacy/v2"}}; + const pending=waitForAuthority(f.execute,"Ready",3); + setTimeout(()=>{f.objects[key].status.phase="Ready"},2100); + await vi.runAllTimersAsync(); + await pending; + } finally { vi.useRealTimers(); } + }); + + it("stages legacy template installations using owned resources and controller CAS without overwriting other env",async()=>{ + const f=fixture(); + const controller=f.objects["deployment/kars-system/kars-controller"]; + controller.spec={template:{spec:{serviceAccountName:"kars-controller",containers:[ + {name:"controller",image:"old/controller:latest",env:[{name:"PRESERVED",value:"setting"}]}, + ]}}}; + const execute=vi.fn(async(file,args,options)=>{ + if(file==="helm"&&args[0]==="list")return {stdout:"[]"}; + if(file==="helm")return {stdout:JSON.stringify({kind:"CustomResourceDefinition", + apiVersion:"apiextensions.k8s.io/v1",metadata:{name:"karssreregistrations.kars.azure.com"},spec:{scope:"Cluster"}})}; + return f.execute(file,args,options); + }); + delete f.objects["crd//karssreregistrations.kars.azure.com"]; + await stageAuthority(execute,"chart","kars-system","kars","new/controller:latest","new/router:latest",false); + const patch=execute.mock.calls.find(([,args])=>args[0]==="patch")!; + const body=JSON.parse(patch[1][patch[1].indexOf("-p")+1]); + expect(body.metadata).toEqual({uid:"controller",resourceVersion:"1"}); + expect(body.spec.template.spec.containers[0].env).toContainEqual({name:"PRESERVED",value:"setting"}); + expect(body.spec.template.spec.containers[0].env).toContainEqual({name:"INFERENCE_ROUTER_IMAGE",value:"new/router:latest"}); + expect(execute.mock.calls.some(([,args])=>args.includes("--force-conflicts"))).toBe(false); + }); + + it.each([ + {apiGroups:[""],resources:["secrets"],verbs:["watch"]}, + {apiGroups:["*"],resources:["*"],verbs:["watch"]}, + {apiGroups:[""],resources:["secrets"],verbs:["*"]}, + {apiGroups:[""],resources:["pods","secrets"],verbs:["get","watch"]}, + ])("rejects group Secret watch authority before any mutation: %j",async rule=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"watcher",uid:"binding",resourceVersion:"1"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"watcher"}, + subjects:[{kind:"Group",name:"system:serviceaccounts:kars-sre",apiGroup:"rbac.authorization.k8s.io"}]}); + f.objects["clusterrole//watcher"]={metadata:{name:"watcher",uid:"role",resourceVersion:"1"},rules:[rule]}; + await expect(assertSafeMutation(f.execute)).rejects.toThrow("broad group grant"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + }); + + it("exempts only the exact ordinary non-Secret spawner role, not a same-name watch grant",async()=>{ + const f=fixture(); + f.bindings.push({kind:"ClusterRoleBinding",metadata:{name:"spawner",uid:"binding",resourceVersion:"1"}, + roleRef:{apiGroup:"rbac.authorization.k8s.io",kind:"ClusterRole",name:"kars-sandbox-spawner"}, + subjects:[{kind:"ServiceAccount",namespace:"kars-sre",name:"sandbox"}]}); + const role={metadata:{name:"kars-sandbox-spawner",uid:"role",resourceVersion:"1"}, + rules:[{apiGroups:["kars.azure.com"],resources:["karssandboxes"],verbs:["get","list","create","delete"]}]}; + f.objects["clusterrole//kars-sandbox-spawner"]=role; + await assertSafeMutation(f.execute); + role.rules=[{apiGroups:[""],resources:["secrets"],verbs:["watch"]}]; + await expect(assertSafeMutation(f.execute)).rejects.toThrow("Legacy SRE grants"); + expect(f.execute.mock.calls.every(([,args])=>args[0]==="get")).toBe(true); + }); +}); diff --git a/cli/src/lib/sre-authority.ts b/cli/src/lib/sre-authority.ts new file mode 100644 index 000000000..fedffa8b2 --- /dev/null +++ b/cli/src/lib/sre-authority.ts @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { namespaceClaimed, type OwnershipObject } from "./namespace-ownership.js"; + +export type Execute = (file: string, args: readonly string[], options: { stdio: "pipe"; input?: string }) => Promise<{ stdout: string }>; +const RESOURCE = "karssreregistrations.kars.azure.com"; +const CRD = "karssreregistrations.kars.azure.com"; +const NS = "kars-sre"; +const OLD_USER = `system:serviceaccount:${NS}:sandbox`; +const PRIVACY_REVISION = "kars.azure.com/sre-privacy/v2"; + +export interface ApiObject extends OwnershipObject { + metadata: OwnershipObject["metadata"] & { generation?: number }; + spec?: Record; + status?: Record; + roleRef?: { apiGroup: string; kind: string; name: string }; + subjects?: Array<{ kind: string; name: string; namespace?: string; apiGroup?: string }>; + rules?: Array<{ apiGroups?: string[]; resources?: string[]; verbs?: string[]; nonResourceURLs?: string[] }>; +} + +export interface Enrollment { + controller: { namespace: { name: string; uid: string }; deployment: { name: string; uid: string }; release: string }; + sandbox: { namespace: string; name: string; uid: string }; + runtimeNamespace: { name: string; uid: string }; + enabled: boolean; + legacyBindings: Array<{ + kind: string; namespace?: string; name: string; uid: string; resourceVersion: string; + roleRef: NonNullable; subjects: NonNullable; + }>; + legacyConsumer?: { namespace: string; name: string; uid: string; resourceVersion: string }; +} + +export async function get(execute: Execute, kind: string, name: string, namespace?: string): Promise { + const { stdout } = await execute("kubectl", ["get", kind, name, ...(namespace ? ["-n", namespace] : []), + "--ignore-not-found", "-o", "json"], { stdio: "pipe" }); + if (!stdout.trim()) return undefined; + const object = JSON.parse(stdout) as ApiObject; + if (!object.metadata?.uid || !object.metadata.resourceVersion || !object.metadata.name) { + throw new Error("SRE authority API response omitted its exact identity"); + } + return object; +} + +async function list(execute: Execute, kind: string): Promise { + const { stdout } = await execute("kubectl", ["get", kind, "-A", "-o", "json"], { stdio: "pipe" }); + const result = JSON.parse(stdout) as { items?: ApiObject[] }; + if (!Array.isArray(result.items)) throw new Error("SRE authority inventory is malformed"); + for (const item of result.items) if (!item.metadata?.uid || !item.metadata.resourceVersion || !item.metadata.name) { + throw new Error("SRE authority inventory omitted an object identity"); + } + return result.items; +} + +export function oldSubject(subject: NonNullable[number]): boolean { + return (subject.kind === "ServiceAccount" && subject.name === "sandbox" && subject.namespace === NS) + || (subject.kind === "User" && subject.name === OLD_USER); +} + +function oldGroup(subject: NonNullable[number]): boolean { + return subject.kind === "Group" + && ["system:authenticated", "system:serviceaccounts", `system:serviceaccounts:${NS}`].includes(subject.name); +} + +function dangerous(role: ApiObject): boolean { + const includes = (items: string[] | undefined, value: string) => items?.some(item => item === "*" || item === value); + return role.rules?.some(rule => + (includes(rule.apiGroups, "") && includes(rule.resources, "secrets") + && ["get", "list", "watch"].some(verb => includes(rule.verbs, verb))) + || (includes(rule.apiGroups, "") && ["pods/exec", "pods/proxy", "serviceaccounts/token"] + .some(resource => includes(rule.resources, resource)) + && (includes(rule.verbs, "get") || includes(rule.verbs, "create"))) + || (includes(rule.apiGroups, "kars.azure.com") && includes(rule.resources, "karssreactions") && includes(rule.verbs, "create")), + ) ?? false; +} + +export async function legacyBindings(execute: Execute): Promise { + const bindings = [...await list(execute, "clusterrolebindings"), ...await list(execute, "rolebindings")]; + const result: ApiObject[] = []; + for (const binding of bindings) { + if (!binding.roleRef) throw new Error("RBAC binding omitted roleRef"); + const subjects = binding.subjects ?? []; + if (subjects.some(oldGroup)) { + const role = await get(execute, binding.roleRef.kind.toLowerCase(), binding.roleRef.name, + binding.roleRef.kind === "Role" ? binding.metadata.namespace : undefined); + if (!role) throw new Error("A referenced RBAC role is missing"); + if (dangerous(role)) throw new Error("A broad group grant gives the legacy SRE identity privileged access; restructure it explicitly before migration"); + } + if (subjects.some(oldSubject)) { + let ordinarySpawner = false; + if (binding.roleRef.kind === "ClusterRole" && binding.roleRef.name === "kars-sandbox-spawner") { + const role = await get(execute, "clusterrole", binding.roleRef.name); + if (!role) throw new Error("A referenced spawner role is missing"); + ordinarySpawner = (role.rules ?? []).every(rule => + JSON.stringify(rule.apiGroups) === '["kars.azure.com"]' + && JSON.stringify(rule.resources) === '["karssandboxes"]' + && !rule.nonResourceURLs + && (rule.verbs ?? []).every(verb => ["get", "list", "create", "delete"].includes(verb))); + } + if (!ordinarySpawner) result.push(binding); + } + } + return result; +} + +export async function requireRegistrar(execute: Execute): Promise { + const { stdout } = await execute("kubectl", ["auth", "can-i", "use", `${RESOURCE}/canonical`], { stdio: "pipe" }); + if (stdout.trim() !== "yes") throw new Error("Explicit cluster-level kars-sre-registrar permission is required"); +} + +export async function preview(execute: Execute, namespace: string, release: string): Promise { + const controllerNs = await get(execute, "namespace", namespace); + const controller = await get(execute, "deployment", "kars-controller", namespace); + const sandbox = await get(execute, "karssandbox", "sre", namespace); + const runtime = await get(execute, "namespace", NS); + if (!controllerNs || !controller || !sandbox || !runtime) throw new Error("Stage the unprivileged canonical SRE source and wait for its namespace claim before enrollment"); + if (controllerNs.metadata.deletionTimestamp || controller.metadata.deletionTimestamp + || sandbox.metadata.deletionTimestamp || runtime.metadata.deletionTimestamp + || !namespaceClaimed(runtime, sandbox) + || sandbox.metadata.annotations?.["kars.azure.com/namespace-uid"] !== runtime.metadata.uid) { + throw new Error("Canonical SRE source/runtime identity or claim is not live and complete"); + } + const annotations = controller.metadata.annotations ?? {}; + if ((annotations["meta.helm.sh/release-name"] && annotations["meta.helm.sh/release-name"] !== release) + || (annotations["meta.helm.sh/release-namespace"] && annotations["meta.helm.sh/release-namespace"] !== namespace)) { + throw new Error("Controller belongs to another release"); + } + const bindings = await legacyBindings(execute); + const consumer = await get(execute, "deployment", "sre", NS); + return { + controller: { + namespace: { name: namespace, uid: controllerNs.metadata.uid! }, + deployment: { name: "kars-controller", uid: controller.metadata.uid! }, release, + }, + sandbox: { namespace, name: "sre", uid: sandbox.metadata.uid! }, + runtimeNamespace: { name: NS, uid: runtime.metadata.uid! }, + enabled: true, + legacyBindings: bindings.map(binding => ({ + kind: binding.kind!, ...(binding.metadata.namespace ? { namespace: binding.metadata.namespace } : {}), + name: binding.metadata.name!, uid: binding.metadata.uid!, resourceVersion: binding.metadata.resourceVersion!, + roleRef: binding.roleRef!, subjects: binding.subjects ?? [], + })), + ...(consumer ? { legacyConsumer: { namespace: NS, name: "sre", uid: consumer.metadata.uid!, + resourceVersion: consumer.metadata.resourceVersion! } } : {}), + }; +} + +export interface Review { + sandboxUid: string; + namespaceUid: string; + binding: string[]; + consumer?: string; + registrationUid?: string; + resourceVersion?: string; +} + +export async function enroll(execute: Execute, spec: Enrollment, review: Review, dryRun: boolean): Promise { + await requireRegistrar(execute); + if (spec.sandbox.uid !== review.sandboxUid || spec.runtimeNamespace.uid !== review.namespaceUid) { + throw new Error("Reviewed SRE source/namespace UID no longer matches"); + } + const expected = spec.legacyBindings.map(binding => + `${binding.kind}/${binding.namespace ?? ""}/${binding.name}=${binding.uid}@${binding.resourceVersion}`); + if (expected.length !== review.binding.length || expected.some(value => !review.binding.includes(value))) { + throw new Error(`Every legacy binding must be explicitly reviewed: ${expected.join(", ")}`); + } + const consumer = spec.legacyConsumer; + if (consumer && review.consumer !== `${consumer.uid}@${consumer.resourceVersion}`) { + throw new Error("Existing SRE consumer requires an exact --consumer UID@resourceVersion review"); + } + const object = { apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSRERegistration", + metadata: { name: "canonical" }, spec }; + const existing = await get(execute, RESOURCE, "canonical"); + if (dryRun) return JSON.stringify(object, null, 2); + if (existing) { + if (existing.metadata.uid !== review.registrationUid || existing.metadata.resourceVersion !== review.resourceVersion) { + throw new Error("Updating an enrollment requires its reviewed registration UID/resourceVersion"); + } + // Enrollment is the complete reviewed spec, not an overlay on retired data. + await execute("kubectl", ["patch", RESOURCE, "canonical", "--type=json", "-p", JSON.stringify([ + { op: "test", path: "/metadata/uid", value: review.registrationUid }, + { op: "test", path: "/metadata/resourceVersion", value: review.resourceVersion }, + { op: "replace", path: "/spec", value: spec }, + ])], { stdio: "pipe" }); + } else { + if (review.registrationUid || review.resourceVersion) throw new Error("Reviewed registration disappeared"); + await execute("kubectl", ["create", "-f", "-", "-o", "json"], { stdio: "pipe", input: JSON.stringify(object) }); + } + return "SRE enrollment recorded; the controller will perform the reviewed migration"; +} + +export async function registration(execute: Execute): Promise { + if (!await get(execute, "crd", CRD)) return undefined; + return get(execute, RESOURCE, "canonical"); +} + +/** Read-only normal-deployment gate. Explicit authority staging is a separate + * registrar operation; ordinary deploy paths must never silently retire grants. */ +export async function assertSafeMutation(execute: Execute): Promise { + const reg = await registration(execute); + if (!reg && !await get(execute, "namespace", NS)) return; + const bindings = await legacyBindings(execute); + if (bindings.length) throw new Error("Legacy SRE grants require 'kars sre authority stage/preview/enroll' before deployment changes"); + if (!reg) return; + if (reg.spec?.enabled === false && reg.status?.phase === "Retired" + && reg.status.observedGeneration === reg.metadata.generation) return; + const spec = reg.spec as unknown as Enrollment; + const current = await preview(execute, spec.controller.namespace.name, spec.controller.release); + if (current.sandbox.uid !== spec.sandbox.uid || current.runtimeNamespace.uid !== spec.runtimeNamespace.uid + || current.controller.deployment.uid !== spec.controller.deployment.uid + || current.controller.namespace.uid !== spec.controller.namespace.uid + || reg.status?.observedGeneration !== reg.metadata.generation + || reg.status?.privacyRevision !== PRIVACY_REVISION + || !["Ready", "Retired"].includes(reg.status?.phase ?? "")) { + throw new Error("SRE authority is stale or migration is incomplete; deployment changes stopped"); + } +} + +export async function assertRollbackSafe(execute: Execute): Promise { + if (await registration(execute)) { + throw new Error("Rollback across enrolled SRE authority is unsafe; use a reviewed roll-forward release instead"); + } + await assertSafeMutation(execute); +} + +export async function waitForAuthority(execute: Execute, phase: "Ready" | "Retired", attempts = 90): Promise { + for (let attempt = 0; attempt < attempts; attempt++) { + const reg = await registration(execute); + if (reg?.status?.phase === phase && reg.status.observedGeneration === reg.metadata.generation + && reg.status.privacyRevision === PRIVACY_REVISION) return; + if (reg?.status?.phase === "Blocked" && reg.status.observedGeneration === reg.metadata.generation) { + throw new Error(`SRE migration blocked: ${String(reg.status.detail ?? "inspect registration status")}`); + } + await new Promise(resolve => setTimeout(resolve, 2000)); + } + throw new Error("SRE authority did not converge; verify the prerequisite controller/router images and registration status"); +} + +export async function retire(execute: Execute, uid: string, resourceVersion: string): Promise { + await requireRegistrar(execute); + const reg = await registration(execute); + if (!reg || reg.metadata.uid !== uid || reg.metadata.resourceVersion !== resourceVersion) { + throw new Error("Reviewed registration UID/resourceVersion changed"); + } + await execute("kubectl", ["patch", RESOURCE, "canonical", "--type=merge", "-p", JSON.stringify({ + metadata: { uid, resourceVersion }, spec: { enabled: false }, + })], { stdio: "pipe" }); + await waitForAuthority(execute, "Retired"); +} + +export async function assertDestroySafe(execute: Execute): Promise { + const reg = await registration(execute); + if (!reg && !await get(execute, "namespace", NS)) return; + if (reg && (reg.status?.phase !== "Retired" || reg.spec?.enabled !== false + || reg.status.observedGeneration !== reg.metadata.generation)) { + throw new Error("Retire the reviewed SRE registration before uninstalling or destroying its controller"); + } + if ((await legacyBindings(execute)).length) throw new Error("Unretired legacy SRE bindings block uninstall/destroy"); +} diff --git a/cli/src/lib/sre-source.ts b/cli/src/lib/sre-source.ts new file mode 100644 index 000000000..88bb3a578 --- /dev/null +++ b/cli/src/lib/sre-source.ts @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { parseAllDocuments } from "yaml"; +import { namespaceClaimed } from "./namespace-ownership.js"; +import { get, legacyBindings, requireRegistrar, type ApiObject, type Execute } from "./sre-authority.js"; + +function same(left: unknown, right: unknown): boolean { + const canonical = (value: any): any => Array.isArray(value) ? value.map(canonical) + : value && typeof value === "object" ? Object.fromEntries(Object.keys(value).sort().map(key => [key,canonical(value[key])])) : value; + return JSON.stringify(canonical(left)) === JSON.stringify(canonical(right)); +} + +/** Stage only unprivileged configuration and CREATE a genuinely new source. + * A 409 never becomes a get/adopt of a racing tenant-created Sandbox. */ +export async function stageSource( + execute: Execute, rendered: string, namespace: string, release: string, +): Promise<{ uid: string; namespaceUid: string }> { + await requireRegistrar(execute); + if ((await legacyBindings(execute)).length) throw new Error("Existing SRE grants require explicit authority preview/enrollment before staging"); + if (await get(execute,"karssandbox","sre",namespace)) { + throw new Error("SRE source already exists; preview and enroll its explicitly reviewed UID instead of adopting by name"); + } + if (await get(execute,"namespace","kars-sre")) { + throw new Error("Existing kars-sre namespace requires explicit ownership review; foreign or unfinished occupants are preserved"); + } + const objects = parseAllDocuments(rendered).map(document => document.toJSON() as ApiObject | null).filter((value): value is ApiObject => !!value); + const source = objects.find(value => value.kind === "KarsSandbox" && value.metadata.name === "sre"); + if (!source || source.metadata.namespace !== namespace) throw new Error("Rendered canonical SRE source is absent or has the wrong workspace"); + const allowed = new Map([ + ["InferencePolicy","sre-inference"], ["ToolPolicy","sre-tools"], + ]); + const supports = objects.filter(value => + allowed.get(value.kind!) === value.metadata.name + || (value.kind === "ClusterRole" && ["kars-sre-reader","kars-sre-action-author","kars-sre-approver"].includes(value.metadata.name!))); + const missing: ApiObject[] = []; + // Validate the entire support set before writing any of it. + for (const object of supports) { + const existing = await get(execute,object.kind!.toLowerCase(),object.metadata.name!,object.metadata.namespace); + if (existing) { + if (!same(existing.spec ?? existing.rules,object.spec ?? object.rules)) { + throw new Error(`Existing ${object.kind}/${object.metadata.name} differs; no support resource was adopted or overwritten`); + } + } else missing.push(object); + } + const own = (object: ApiObject) => ({ + ...object, metadata: { + ...object.metadata, + labels: { ...object.metadata.labels, "app.kubernetes.io/managed-by":"Helm" }, + annotations: { ...object.metadata.annotations, + "meta.helm.sh/release-name":release,"meta.helm.sh/release-namespace":namespace }, + }, + }); + for (const object of missing) { + await execute("kubectl",["create","-f","-"],{stdio:"pipe",input:JSON.stringify(own(object))}); + } + const {stdout}=await execute("kubectl",["create","-f","-","-o","json"],{stdio:"pipe",input:JSON.stringify(own(source))}); + const created=JSON.parse(stdout) as ApiObject; + if (created.metadata?.namespace!==namespace || created.metadata.name!=="sre" || !created.metadata.uid) { + throw new Error("SRE CREATE response omitted its actual source UID/workspace"); + } + for (let attempt=0;attempt<60;attempt++) { + const live=await get(execute,"karssandbox","sre",namespace); + if (!live || live.metadata.uid!==created.metadata.uid || live.metadata.deletionTimestamp) { + throw new Error("Created SRE source was replaced or deleted during namespace staging"); + } + const runtime=await get(execute,"namespace","kars-sre"); + if (runtime) { + if (runtime.metadata.deletionTimestamp || !namespaceClaimed(runtime,live)) { + throw new Error("SRE runtime namespace belongs to a foreign/ambiguous occupant; preserved without any grant"); + } + if (live.metadata.annotations?.["kars.azure.com/namespace-uid"]===runtime.metadata.uid) { + return {uid:created.metadata.uid,namespaceUid:runtime.metadata.uid!}; + } + } + await new Promise(resolve=>setTimeout(resolve,1000)); + } + throw new Error("SRE source was staged but its exact namespace claim did not converge"); +} diff --git a/cli/src/lib/sre-stage.ts b/cli/src/lib/sre-stage.ts new file mode 100644 index 000000000..30871afa9 --- /dev/null +++ b/cli/src/lib/sre-stage.ts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { parseAllDocuments } from "yaml"; +import { get, requireRegistrar, type ApiObject, type Execute } from "./sre-authority.js"; + +function parts(image: string): [string,string] { + const index=image.lastIndexOf(":"); + if(index<=image.lastIndexOf("/")||image.includes("@"))throw new Error("Stage images require explicit repository:tag references"); + return [image.slice(0,index),image.slice(index+1)]; +} + +function canonical(value:any):string { + const sort=(value:any):any=>Array.isArray(value)?value.map(sort) + :value&&typeof value==="object"?Object.fromEntries(Object.keys(value).sort().map(key=>[key,sort(value[key])])):value; + return JSON.stringify(sort(value)); +} + +export async function stageAuthority( + execute:Execute,chart:string,namespace:string,release:string, + controllerImage:string,routerImage:string,dryRun:boolean, +):Promise { + await requireRegistrar(execute); + const controller=await get(execute,"deployment","kars-controller",namespace); + if(!controller)throw new Error("Install the core prerequisite first; authority staging does not provision a new cluster"); + if(controller.spec?.template?.spec?.serviceAccountName!=="kars-controller") { + throw new Error("Controller uses a custom ServiceAccount; review and stage its minimal authority role explicitly"); + } + const {stdout}=await execute("helm",["list","-n",namespace,"--all","-o","json"],{stdio:"pipe"}); + const releases=JSON.parse(stdout) as unknown; + if(!Array.isArray(releases)||releases.some(item=>!item||typeof item.name!=="string"||item.namespace!==namespace)) { + throw new Error("Helm ownership inventory is invalid"); + } + const [controllerRepository,controllerTag]=parts(controllerImage); + const [routerRepository,routerTag]=parts(routerImage); + if(releases.some(item=>item.name===release)) { + await execute("helm",["upgrade",release,chart,"--namespace",namespace,"--reset-then-reuse-values", + "--set","sre.authorityStage=true", + "--set-string",`controller.image.repository=${controllerRepository}`, + "--set-string",`controller.image.tag=${controllerTag}`, + "--set-string",`inferenceRouter.image.repository=${routerRepository}`, + "--set-string",`inferenceRouter.image.tag=${routerTag}`, + ...(dryRun?["--dry-run=server"]:["--wait","--timeout","8m"])],{stdio:"pipe"}); + return; + } + if(controller.metadata.annotations?.["meta.helm.sh/release-name"]) { + throw new Error("Controller reports Helm ownership that was not found; no template-mode adoption is allowed"); + } + const rendered=await execute("helm",["template",release,chart,"--namespace",namespace, + "--set","sre.enabled=false","--set","azure.workloadIdentity.clientId=dummy"],{stdio:"pipe"}); + const allowedRoles=["kars-sre-registrar","kars-sre-router-renew","kars-sre-private-diagnostics","kars-sre-retired-agent","kars-sre-authority-controller"]; + const objects=parseAllDocuments(rendered.stdout).map(doc=>doc.toJSON() as ApiObject|null).filter((obj):obj is ApiObject=>!!obj) + .filter(obj=>(obj.kind==="CustomResourceDefinition"&&obj.metadata.name==="karssreregistrations.kars.azure.com") + || (["ValidatingAdmissionPolicy","ValidatingAdmissionPolicyBinding"].includes(obj.kind!)&&obj.metadata.name?.startsWith("kars-sre-")) + || (obj.kind==="ClusterRole"&&allowedRoles.includes(obj.metadata.name!)) + || (obj.kind==="ClusterRoleBinding"&&obj.metadata.name==="kars-sre-authority-controller")); + if(!objects.some(obj=>obj.kind==="CustomResourceDefinition"))throw new Error("Authority CRD is absent from the staged chart"); + const writes:Array<{object:ApiObject;existing?:ApiObject}>=[]; + for(const object of objects) { + const existing=await get(execute,object.kind!.toLowerCase(),object.metadata.name!); + if(existing) { + const desired=object.spec??object.rules??{roleRef:object.roleRef,subjects:object.subjects}; + const current=existing.spec??existing.rules??{roleRef:existing.roleRef,subjects:existing.subjects}; + if(canonical(desired)===canonical(current))continue; + if(existing.metadata.annotations?.["kars.azure.com/sre-authority-staged"]!==namespace + || existing.metadata.annotations?.["kars.azure.com/sre-authority-release"]!==release) { + throw new Error(`Unowned authority object ${object.kind}/${object.metadata.name} differs; no objects were adopted`); + } + } + writes.push({object,existing}); + } + const containers=structuredClone(controller.spec?.template?.spec?.containers); + if(!Array.isArray(containers))throw new Error("Controller container specification is missing"); + const main=containers.find(container=>container.name==="controller"); + if(!main)throw new Error("Controller container identity is unrecognized"); + main.image=controllerImage; + main.env=(main.env??[]).filter((entry:{name:string})=>entry.name!=="INFERENCE_ROUTER_IMAGE"); + main.env.push({name:"INFERENCE_ROUTER_IMAGE",value:routerImage}); + if(dryRun) { + console.log(`Would stage ${writes.length} authority objects and CAS-update controller ${controller.metadata.uid}@${controller.metadata.resourceVersion}`); + return; + } + for(const {object,existing} of writes.sort((a,b)=>Number(b.object.kind==="CustomResourceDefinition")-Number(a.object.kind==="CustomResourceDefinition"))) { + const annotations={...object.metadata.annotations, + "kars.azure.com/sre-authority-staged":namespace,"kars.azure.com/sre-authority-release":release}; + if(existing) { + await execute("kubectl",["patch",object.kind!.toLowerCase(),object.metadata.name!,"--type=merge","-p",JSON.stringify({ + ...object,metadata:{uid:existing.metadata.uid,resourceVersion:existing.metadata.resourceVersion,annotations}, + })],{stdio:"pipe"}); + } else { + await execute("kubectl",["create","-f","-"],{stdio:"pipe",input:JSON.stringify({ + ...object,metadata:{...object.metadata,annotations}, + })}); + } + if(object.kind==="CustomResourceDefinition") { + await execute("kubectl",["wait","--for=condition=Established",`crd/${object.metadata.name}`,"--timeout=60s"],{stdio:"pipe"}); + } + } + await execute("kubectl",["patch","deployment","kars-controller","-n",namespace,"--type=merge","-p",JSON.stringify({ + metadata:{uid:controller.metadata.uid,resourceVersion:controller.metadata.resourceVersion}, + spec:{template:{spec:{containers}}}, + })],{stdio:"pipe"}); + await execute("kubectl",["rollout","status","deployment/kars-controller","-n",namespace,"--timeout=8m"],{stdio:"pipe"}); +} diff --git a/cli/src/testing/shared-security-guards.test.ts b/cli/src/testing/shared-security-guards.test.ts new file mode 100644 index 000000000..6b02ce7f1 --- /dev/null +++ b/cli/src/testing/shared-security-guards.test.ts @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const root = new URL("../../../", import.meta.url); +const source = (path: string) => readFileSync(new URL(path, root), "utf8"); + +describe("shared Rust security guard coverage", () => { + it.each(["no-stubs.sh", "no-custom-crypto.sh"])("includes shared code in %s production paths", name => { + const paths = source(`ci/${name}`).match(/PROD_PATHS=\(([\s\S]*?)\n\)/)?.[1]; + expect(paths).toBeDefined(); + expect(paths).toContain("'shared/'"); + }); + + it("requires a capability audit for shared Rust changes without gating Markdown", () => { + const pattern = source("ci/security-audit-required.sh").match(/^CAP_RE='([^']+)'$/m)?.[1]; + expect(pattern).toBeDefined(); + const result = spawnSync("grep", ["-E", pattern!], { + encoding: "utf8", + input: [ + "shared/sre_privacy.rs", "shared/future/authority.rs", + "shared/README.md", "docs/shared/sre_privacy.rs", + ].join("\n"), + }); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "shared/sre_privacy.rs", "shared/future/authority.rs", + ]); + }); +}); diff --git a/cli/src/testing/sre-authority.test.ts b/cli/src/testing/sre-authority.test.ts new file mode 100644 index 000000000..6599b6de4 --- /dev/null +++ b/cli/src/testing/sre-authority.test.ts @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe,expect,it } from "vitest"; +import { parseAllDocuments } from "yaml"; + +const root=fileURLToPath(new URL("../../../",import.meta.url)); +const chart=fileURLToPath(new URL("../../../deploy/helm/kars",import.meta.url)); + +describe("SRE authority chart and mutation integration",()=>{ + it("creates a cluster registration and no default registrar or runtime privilege bindings",()=>{ + for(const enabled of [false,true]){ + const output=execFileSync("helm",["template","kars",chart,"--namespace","kars-system","--set",`sre.enabled=${enabled}`],{encoding:"utf8"}); + const docs=parseAllDocuments(output).map(doc=>doc.toJSON()).filter(Boolean); + const registration=docs.find(doc=>doc.kind==="CustomResourceDefinition"&&doc.metadata.name==="karssreregistrations.kars.azure.com"); + expect(registration.spec.scope).toBe("Cluster"); + const bindings=docs.filter(doc=>["ClusterRoleBinding","RoleBinding"].includes(doc.kind)); + expect(bindings.some(binding=>binding.roleRef.name==="kars-sre-registrar")).toBe(false); + expect(bindings.some(binding=>(binding.subjects??[]).some((subject:any)=> + subject.namespace==="kars-sre"&&["sandbox","sre-api-router"].includes(subject.name)))).toBe(false); + const controller=docs.find(doc=>doc.kind==="ClusterRole"&&doc.metadata.name==="kars-sre-authority-controller"); + expect(controller.rules.filter((rule:any)=>rule.resources.includes("karssreregistrations")) + .every((rule:any)=>rule.verbs.every((verb:string)=>["get","list","watch","use"].includes(verb)))).toBe(true); + } + }); + + it("uses authorizer permissions rather than usernames for protected source/identity admission",()=>{ + const output=execFileSync("helm",["template","kars",chart,"--namespace","custom-controller"],{encoding:"utf8"}); + const policies=parseAllDocuments(output).map(doc=>doc.toJSON()).filter(doc=>doc?.kind==="ValidatingAdmissionPolicy"&&doc.metadata.name.startsWith("kars-sre-")); + expect(policies.length).toBeGreaterThan(5); + for(const name of ["kars-sre-source-authority","kars-sre-private-identity","kars-sre-registration-authority"]){ + const policy=policies.find(doc=>doc.metadata.name===name); + expect(policy.spec.failurePolicy).toBe("Fail"); + const expressions=JSON.stringify(policy.spec); + expect(expressions).toContain("authorizer.group('kars.azure.com')"); + expect(expressions).not.toContain("request.userInfo.username"); + } + }); + + it("places normal mutation and rollback checks before their owning writes",()=>{ + const checks=[ + ["cli/src/commands/push-apply.ts","await assertSafeMutation(execute)","const artifacts ="], + ["cli/src/commands/up/fast_upgrade.ts","await assertSafeMutation(execa)","const helmArgs ="], + ["cli/src/commands/up.ts","await assertSafeMutation(execa)","const helmArgs ="], + ["cli/src/commands/dev/local-k8s.ts","await assertSafeMutation(","const credsOverlay = await provisionDevCreds"], + ["cli/src/commands/upgrade.ts","await assertRollbackSafe(execa)","await execa(\"helm\", [\"rollback\""], + ]; + for(const [path,gate,write] of checks){ + const source=readFileSync(`${root}/${path}`,"utf8"); + expect(source.indexOf(gate),path).toBeGreaterThanOrEqual(0); + expect(source.indexOf(gate),path).toBeLessThan(source.indexOf(write)); + } + }); + + it("unconditionally denies legacy token Secret creation and both old/new update transitions under arbitrary names",()=>{ + const output=execFileSync("helm",["template","kars",chart],{encoding:"utf8"}); + const docs=parseAllDocuments(output).map(doc=>doc.toJSON()).filter(Boolean); + const policy=docs.find(doc=>doc.kind==="ValidatingAdmissionPolicy"&&doc.metadata.name==="kars-sre-no-legacy-tokens"); + expect(policy.spec.matchConstraints.resourceRules).toEqual([{ + apiGroups:[""],apiVersions:["v1"],operations:["CREATE","UPDATE"],resources:["secrets"], + }]); + const expression=policy.spec.validations[0].expression; + expect(expression).toContain("oldObject"); + expect(expression).toContain("object.?type"); + expect(expression).toContain("kubernetes.io/service-account.name"); + expect(expression).toContain("kubernetes.io/service-account-token"); + expect(expression).not.toContain("metadata.name"); + expect(expression).not.toContain("authorizer"); + expect(docs.find(doc=>doc.kind==="ValidatingAdmissionPolicyBinding" + &&doc.metadata.name===policy.metadata.name).spec.validationActions).toContain("Deny"); + }); +}); diff --git a/controller/Cargo.toml b/controller/Cargo.toml index 3a359e94f..1216bee75 100644 --- a/controller/Cargo.toml +++ b/controller/Cargo.toml @@ -68,6 +68,8 @@ oci-client = { version = "=0.16.1", default-features = false, features = ["rustl # refuse to auto-detect and panic on first TLS handshake. Pin to # `aws-lc-rs` to align with the rest of the workspace. rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } +rcgen.workspace = true +time.workspace = true regex = "1.12.3" [dev-dependencies] diff --git a/controller/src/main.rs b/controller/src/main.rs index 6a8a9bb4f..01ff8dbdb 100644 --- a/controller/src/main.rs +++ b/controller/src/main.rs @@ -73,6 +73,10 @@ mod policy_fetcher; mod providers; mod reconciler; mod signer_policy; +mod sre_authority; +#[path = "../../shared/sre_privacy.rs"] +mod sre_privacy; +mod sre_registration; mod status; mod task_models; mod team_commons; @@ -299,6 +303,10 @@ async fn main() -> Result<()> { let client = client.clone(); tokio::spawn(async move { kars_sre_action_reconciler::run(client).await }) }; + let sre_authority_handle = { + let client = client.clone(); + tokio::spawn(async move { sre_authority::run(client).await }) + }; let auth_config_handle = { // KarsAuthConfig reconciler — materialises the sidecar env // ConfigMap when an operator installs the tenant trust anchor @@ -458,6 +466,9 @@ async fn main() -> Result<()> { res = kars_sre_action_handle => { res??; } + res = sre_authority_handle => { + res?; + } res = auth_config_handle => { // auth-config reconciler exiting is non-fatal (it sleeps // forever when the CRD is absent), but we propagate any diff --git a/controller/src/providers/mod.rs b/controller/src/providers/mod.rs index e64ac9a9d..fcb4259a0 100644 --- a/controller/src/providers/mod.rs +++ b/controller/src/providers/mod.rs @@ -29,6 +29,7 @@ /// Governance Receipt signing (kars Bridge Inc 3). Allowlisted crypto /// wrapper: Ed25519 over DSSE. See the module docs for the V0 trust model. pub mod signing; +pub mod sre_tls; #[allow(unused_imports)] pub mod field_managers { diff --git a/controller/src/providers/signing.rs b/controller/src/providers/signing.rs index bcc0e5f05..c1e481932 100644 --- a/controller/src/providers/signing.rs +++ b/controller/src/providers/signing.rs @@ -177,6 +177,13 @@ pub fn content_digest(bytes: &[u8]) -> String { format!("sha256:{}", &sha256_hex(bytes)[..32]) } +/// Opaque service credentials; callers must keep them out of agent authority +/// except where the credential deliberately grants only a filtered interface. +pub fn generate_service_token() -> String { + use rand::distr::{Alphanumeric, SampleString}; + Alphanumeric.sample_string(&mut rand::rng(), 64) +} + /// DSSE Pre-Authentication Encoding: /// `"DSSEv1" SP len(type) SP type SP len(body) SP body`. /// diff --git a/controller/src/providers/sre_tls.rs b/controller/src/providers/sre_tls.rs new file mode 100644 index 000000000..bfd1e5d80 --- /dev/null +++ b/controller/src/providers/sre_tls.rs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Standard X.509 issuance for a pod-local compatibility endpoint. + +use rcgen::{ + BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose, +}; + +pub struct Identity { + pub ca: String, + pub certificate: String, + pub private_key: String, + pub expires_at: i64, +} + +pub fn issue() -> Result { + let now = time::OffsetDateTime::now_utc(); + let not_before = now - time::Duration::hours(1); + let expiry = now + time::Duration::days(30); + let mut root = CertificateParams::new(Vec::::new()) + .map_err(|_| "SRE CA parameters are invalid")?; + root.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + root.distinguished_name + .push(rcgen::DnType::CommonName, "Kars SRE loopback CA"); + root.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + root.not_before = not_before; + root.not_after = expiry; + let root_key = KeyPair::generate().map_err(|_| "SRE CA key generation failed")?; + let ca = root + .self_signed(&root_key) + .map_err(|_| "SRE CA issuance failed")?; + let mut leaf = CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]) + .map_err(|_| "SRE TLS parameters are invalid")?; + leaf.not_before = not_before; + leaf.distinguished_name + .push(rcgen::DnType::CommonName, "Kars SRE loopback API"); + leaf.not_after = expiry; + leaf.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + leaf.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + let key = KeyPair::generate().map_err(|_| "SRE TLS key generation failed")?; + let certificate = leaf + .signed_by(&key, &ca, &root_key) + .map_err(|_| "SRE TLS issuance failed")?; + Ok(Identity { + ca: ca.pem(), + certificate: certificate.pem(), + private_key: key.serialize_pem(), + expires_at: expiry.unix_timestamp(), + }) +} diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 8e86917fe..75d1b7d66 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -199,22 +199,8 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result projection, + Err(detail) => { + crate::status::stamp_degraded(client, &sandbox, &name, "SREAuthorityNotReady", &detail) + .await; + return Ok(Action::requeue(Duration::from_secs(20))); + } + }; let credentials = credential_sources::reconcile( client, &sandbox, @@ -942,20 +937,8 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, + >(annotations)?); + } if let Some(namespace) = owned_namespace.as_ref() { credentials.decorate(&mut deployment, &sandbox, namespace); } @@ -3348,6 +3330,16 @@ pub async fn run(client: Client) -> Result<()> { crate::watch_config::bounded(), deployment_to_sandbox_ref, ) + .watches( + Api::::all(ctx.client.clone()), + crate::watch_config::bounded(), + |registration| { + Some( + kube::runtime::reflector::ObjectRef::new(®istration.spec.sandbox.name) + .within(®istration.spec.sandbox.namespace), + ) + }, + ) .run( |x, ctx| async move { crate::metrics::observe_reconcile("KarsSandbox", reconcile(x, ctx)).await diff --git a/controller/src/reconciler/pod_spec.rs b/controller/src/reconciler/pod_spec.rs index 0e4c70e3a..22925e184 100644 --- a/controller/src/reconciler/pod_spec.rs +++ b/controller/src/reconciler/pod_spec.rs @@ -94,28 +94,14 @@ pub(crate) fn sandbox_node_selector_from( /// Build the egress-guard init-container command. /// -/// Standard sandboxes (every kind except SRE) get the full lockdown: +/// Every sandbox, including SRE, gets the full lockdown: /// UID 1000 → loopback + DNS allowed, everything else dropped, with /// :80/:443 NAT-redirected to the inference-router on :8444 for L7 /// policy + audit. /// -/// SRE-mode sandboxes (labelled `kars.azure.com/role=sre`) get ONE -/// extra rule inserted into the OUTPUT NAT chain BEFORE the generic -/// REDIRECT: apiserver-bound traffic (KUBERNETES_SERVICE_HOST : -/// KUBERNETES_SERVICE_PORT_HTTPS, both kubelet-auto-injected envs) -/// is RETURNed — i.e. NOT NAT'd to :8444 — so the SRE plugin's K8s -/// API client (sre_kube.py) can hit the apiserver directly with its -/// projected SA token. -/// -/// The K8s audit log is the audit surface for these apiserver calls -/// (the router's L7 audit doesn't capture them, but K8s audit is -/// stronger — every call carries the SA identity and the verb). -/// -/// Privilege-containment design: this capability is uniquely held by -/// the SRE sandbox per the proposal §7.8. Future Slice 3 will add -/// ValidatingAdmissionPolicies to gate WHO can apply the -/// `role=sre` label (only chart-installer SAs; see §7.8.10 design). -pub(crate) fn build_egress_guard_command(is_sre_sandbox: bool) -> String { +/// SRE diagnostics use a registered, filtered loopback HTTPS service instead +/// of direct Kubernetes access with an agent-held privileged credential. +pub(crate) fn build_egress_guard_command(_is_sre_sandbox: bool) -> String { let mut cmd = String::with_capacity(1024); // Filter chain (OUTPUT): UID 1000 → allow loopback + DNS + // established, then DROP. Same for every sandbox kind. @@ -126,36 +112,8 @@ pub(crate) fn build_egress_guard_command(is_sre_sandbox: bool) -> String { "iptables -A OUTPUT -m owner --uid-owner 1000 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT && " ); - // SRE-mode-only: filter-chain ACCEPT for apiserver-bound traffic. - // The filter chain runs AFTER the NAT chain — the NAT-bypass RETURN - // below just decides "don't redirect", but the filter chain's DROP - // (next rule) would still kill the packet. We have to ACCEPT it - // here BEFORE the catch-all DROP. - if is_sre_sandbox { - cmd.push_str( - "iptables -A OUTPUT -m owner --uid-owner 1000 \ - -d \"${KUBERNETES_SERVICE_HOST}\" \ - -p tcp --dport \"${KUBERNETES_SERVICE_PORT_HTTPS:-443}\" \ - -j ACCEPT && ", - ); - } - cmd.push_str("iptables -A OUTPUT -m owner --uid-owner 1000 -j DROP && "); - // SRE-mode-only: NAT-chain apiserver bypass. Inserted BEFORE the - // generic :443 REDIRECT so apiserver traffic short-circuits to the - // real upstream rather than the router. KUBERNETES_SERVICE_HOST - // and KUBERNETES_SERVICE_PORT_HTTPS are auto-injected by the - // kubelet on every container (including init containers). - if is_sre_sandbox { - cmd.push_str( - "iptables -t nat -A OUTPUT -m owner --uid-owner 1000 \ - -d \"${KUBERNETES_SERVICE_HOST}\" \ - -p tcp --dport \"${KUBERNETES_SERVICE_PORT_HTTPS:-443}\" \ - -j RETURN && ", - ); - } - // NAT chain (OUTPUT): :80/:443 → REDIRECT to :8444 (transparent // proxy in the inference-router sidecar). Same for every sandbox. cmd.push_str( @@ -165,15 +123,7 @@ pub(crate) fn build_egress_guard_command(is_sre_sandbox: bool) -> String { "iptables -t nat -A OUTPUT -m owner --uid-owner 1000 ! -o lo -p tcp --dport 443 -j REDIRECT --to-port 8444 && " ); - if is_sre_sandbox { - cmd.push_str( - "echo 'egress-guard: UID 1000 → transparent proxy on :8444 + apiserver bypass (SRE mode)'" - ); - } else { - cmd.push_str( - "echo 'egress-guard: UID 1000 → transparent proxy on :8444 (learn + enforce)'", - ); - } + cmd.push_str("echo 'egress-guard: UID 1000 → transparent proxy on :8444 (learn + enforce)'"); cmd } @@ -193,53 +143,11 @@ mod egress_guard_tests { } #[test] - fn sre_sandbox_inserts_apiserver_bypass_before_redirect() { + fn sre_uses_the_same_network_lockdown_without_an_apiserver_bypass() { let cmd = build_egress_guard_command(true); - // The bypass MUST come before the :443 REDIRECT — otherwise - // the REDIRECT wins (iptables -A appends; rules evaluate in - // order) and the bypass is dead code. - let bypass_pos = cmd - .find("-t nat -A OUTPUT -m owner --uid-owner 1000 -d \"${KUBERNETES_SERVICE_HOST}\"") - .or_else(|| cmd.find("-t nat -A OUTPUT -m owner --uid-owner 1000 \t\t\t -d \"${KUBERNETES_SERVICE_HOST}\"")) - .or_else(|| { - // Match the NAT-chain bypass specifically (not the filter ACCEPT) - cmd.match_indices("-t nat -A OUTPUT") - .find(|(i, _)| cmd[*i..].contains("KUBERNETES_SERVICE_HOST")) - .map(|(i, _)| i) - }) - .expect("NAT-chain bypass rule missing"); - let redirect_pos = cmd - .find("--dport 443 -j REDIRECT") - .expect("redirect rule missing"); - assert!( - bypass_pos < redirect_pos, - "NAT bypass at {bypass_pos} must precede redirect at {redirect_pos}" - ); - assert!(cmd.contains("apiserver bypass (SRE mode)")); - - // ALSO check the filter-chain ACCEPT exists BEFORE the DROP — this - // was the bug we hit live: NAT bypass alone wasn't enough because - // the filter chain's DROP for UID 1000 killed the packet anyway. - let filter_accept = cmd - .find( - "-A OUTPUT -m owner --uid-owner 1000 -d \"${KUBERNETES_SERVICE_HOST}\"", - ) - .or_else(|| { - cmd.match_indices("-A OUTPUT -m owner --uid-owner 1000") - .find(|(i, _)| { - let tail = &cmd[*i..*i + 200.min(cmd.len() - *i)]; - tail.contains("KUBERNETES_SERVICE_HOST") && tail.contains("-j ACCEPT") - }) - .map(|(i, _)| i) - }) - .expect("filter-chain ACCEPT for apiserver missing"); - let filter_drop = cmd - .find("-A OUTPUT -m owner --uid-owner 1000 -j DROP") - .expect("filter DROP rule missing"); - assert!( - filter_accept < filter_drop, - "filter ACCEPT at {filter_accept} must precede DROP at {filter_drop}" - ); + assert_eq!(cmd, build_egress_guard_command(false)); + assert!(!cmd.contains("KUBERNETES_SERVICE_HOST")); + assert!(!cmd.contains("-j RETURN")); } #[test] diff --git a/controller/src/sre_authority.rs b/controller/src/sre_authority.rs new file mode 100644 index 000000000..69c53eaf3 --- /dev/null +++ b/controller/src/sre_authority.rs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Registrar-authorized migration and private SRE Kubernetes identity. + +mod admission; +mod bindings; +mod credential_guard; +mod credentials; +mod live; +mod migration; +pub(crate) mod pod; +#[cfg(test)] +mod privacy_tests; +#[cfg(test)] +mod tests; + +use crate::sre_registration::{KarsSRERegistration, NAME, RegistrationStatus}; +use kube::{ + Api, Client, ResourceExt, + api::{Patch, PatchParams}, +}; + +pub(crate) use live::{api_error, check_secret_denial, privacy_epoch}; + +async fn status( + client: &Client, + reg: &KarsSRERegistration, + phase: &str, + detail: Option, + service_account_uid: Option, +) -> Result<(), String> { + let api: Api = Api::all(client.clone()); + let status = RegistrationStatus { + phase: phase.into(), + observed_generation: reg.metadata.generation.unwrap_or_default(), + privacy_epoch: (phase == "Ready").then(|| reg.epoch()), + router_service_account_uid: if phase == "Retired" { + None + } else { + service_account_uid.or_else(|| { + reg.status + .as_ref() + .and_then(|status| status.router_service_account_uid.clone()) + }) + }, + legacy_secret_access_denied: matches!(phase, "Ready" | "Retired"), + privacy_revision: matches!(phase, "Ready" | "Retired") + .then(|| crate::sre_privacy::REVISION.into()), + detail, + }; + let mut status_value = + serde_json::to_value(&status).map_err(|_| "SRE authority status serialization failed")?; + status_value["privacyEpoch"] = serde_json::json!(status.privacy_epoch); + status_value["privacyRevision"] = serde_json::json!(status.privacy_revision); + status_value["routerServiceAccountUid"] = serde_json::json!(status.router_service_account_uid); + api.patch_status( + NAME, + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "metadata":{"uid":reg.metadata.uid,"resourceVersion":reg.metadata.resource_version}, + "status":status_value, + })), + ) + .await + .map_err(|error| api_error("Publish SRE authority status", error))?; + Ok(()) +} + +pub async fn reconcile(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + reg.validate()?; + let result = reconcile_inner(client, reg).await; + if let Err(error) = &result { + let waiting = migration::is_waiting(error); + let revocation = if waiting { + Ok(()) + } else { + bindings::revoke_private(client, reg).await + }; + let detail = match revocation { + Ok(()) => error.clone(), + Err(revocation) => { + format!("{error}; owned private authority revocation failed: {revocation}") + } + }; + let detail = if !waiting + && reg + .status + .as_ref() + .is_some_and(|status| status.router_service_account_uid.is_some()) + { + match credentials::retire(client, reg).await { + Ok(()) => detail, + Err(error) => { + format!("{detail}; owned private credential retirement failed: {error}") + } + } + } else { + detail + }; + status( + client, + reg, + if waiting { "Migrating" } else { "Blocked" }, + Some(detail), + None, + ) + .await?; + } + result +} + +async fn reconcile_inner(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + reg.validate()?; + if !reg.spec.enabled + && reg.status.as_ref().is_some_and(|status| { + status.phase == "Retired" + && status.observed_generation == reg.metadata.generation.unwrap_or_default() + && status.privacy_revision.as_deref() == Some(crate::sre_privacy::REVISION) + }) + { + return check_secret_denial(client, ®.spec.runtime_namespace.name).await; + } + if !reg.spec.enabled { + migration::stop_registered_consumer_for_retirement(client, reg).await?; + bindings::revoke_private(client, reg).await?; + credentials::retire(client, reg).await?; + check_secret_denial(client, ®.spec.runtime_namespace.name).await?; + return status(client, reg, "Retired", None, None).await; + } + let authority = live::verify(client, reg).await?; + admission::verify(client).await?; + credential_guard::scan(client, reg).await?; + // Validate the full review set before retiring even the first grant. + let reviewed = bindings::review(client, reg).await?; + migration::validate_consumer(client, reg).await?; + bindings::validate_private_targets(client, reg).await?; + credentials::review_targets(client, reg).await?; + bindings::retire_legacy(client, reg, &reviewed).await?; + check_secret_denial(client, ®.spec.runtime_namespace.name).await?; + migration::stop_legacy_consumer(client, reg).await?; + let service_account = credentials::ensure_service_account(client, reg, &authority).await?; + let refreshed = if reg + .status + .as_ref() + .and_then(|status| status.router_service_account_uid.as_ref()) + != service_account.metadata.uid.as_ref() + { + status(client, reg, "Provisioning", None, service_account.uid()).await?; + Api::::all(client.clone()) + .get(NAME) + .await + .map_err(|error| api_error("Refresh provisioning registration", error))? + } else { + reg.clone() + }; + if refreshed.metadata.uid != reg.metadata.uid + || refreshed.metadata.generation != reg.metadata.generation + { + return Err("Registration changed while provisioning private identity".into()); + } + let reg = &refreshed; + bindings::grant_private(client, reg, &service_account).await?; + // No private material is minted until the old principal is actually denied. + check_secret_denial(client, ®.spec.runtime_namespace.name).await?; + credential_guard::scan(client, reg).await?; + live::verify(client, reg).await?; + credentials::ensure(client, reg, &authority, &service_account).await?; + migration::rotate_owned_control_credentials(client, reg).await?; + status(client, reg, "Ready", None, service_account.uid()).await +} + +pub async fn run(client: Client) { + let api: Api = Api::all(client.clone()); + loop { + match api.get_opt(NAME).await { + Ok(Some(reg)) => { + if let Err(error) = reconcile(&client, ®).await { + tracing::warn!(registration = NAME, error = %error, "SRE authority is not ready"); + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!(error = %api_error("Read SRE registration", error), "SRE authority unavailable") + } + } + tokio::time::sleep(std::time::Duration::from_secs(20)).await; + } +} diff --git a/controller/src/sre_authority/admission.rs b/controller/src/sre_authority/admission.rs new file mode 100644 index 000000000..2723ebc33 --- /dev/null +++ b/controller/src/sre_authority/admission.rs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::api_error; +use k8s_openapi::api::admissionregistration::v1::{ + ValidatingAdmissionPolicy, ValidatingAdmissionPolicyBinding, +}; +use kube::{Api, Client}; + +pub(super) const POLICIES: &[&str] = &[ + "kars-sre-source-authority", + "kars-sre-registration-authority", + "kars-sre-private-identity", + "kars-sre-binding-authority", + "kars-sre-private-material", + "kars-sre-no-legacy-tokens", + "kars-sre-source-retirement", + "kars-sre-pending-proposals", + "kars-sre-consumer-authority", + "kars-sre-private-mounts", + "kars-sre-private-workloads", + "kars-sre-private-cronjobs", + "kars-sre-private-connect", + "kars-sre-role-authority", +]; + +pub(super) async fn verify(client: &Client) -> Result<(), String> { + let policies: Api = Api::all(client.clone()); + let bindings: Api = Api::all(client.clone()); + for name in POLICIES { + let policy = policies + .get(name) + .await + .map_err(|e| api_error("Verify SRE admission policy", e))?; + let binding = bindings + .get(name) + .await + .map_err(|e| api_error("Verify SRE admission binding", e))?; + let checked = policy.status.as_ref().is_some_and(|status| { + status.observed_generation == policy.metadata.generation + && status.type_checking.as_ref().is_some_and(|checking| { + checking + .expression_warnings + .as_ref() + .is_none_or(Vec::is_empty) + }) + }); + if policy.metadata.deletion_timestamp.is_some() + || binding.metadata.deletion_timestamp.is_some() + || policy + .spec + .as_ref() + .and_then(|spec| spec.failure_policy.as_deref()) + != Some("Fail") + || !checked + || binding.spec.as_ref().is_none_or(|spec| { + spec.policy_name.as_deref() != Some(*name) + || spec + .validation_actions + .as_ref() + .is_none_or(|actions| !actions.iter().any(|action| action == "Deny")) + }) + { + return Err(format!( + "SRE admission policy {name} is not observed, type-checked, and enforced; no privilege may be issued" + )); + } + } + Ok(()) +} diff --git a/controller/src/sre_authority/bindings.rs b/controller/src/sre_authority/bindings.rs new file mode 100644 index 000000000..086e458ed --- /dev/null +++ b/controller/src/sre_authority/bindings.rs @@ -0,0 +1,623 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::api_error; +use crate::sre_registration::{ + BindingReview, KarsSRERegistration, OWNER, ROUTER_SA, RUNTIME_NAMESPACE, +}; +use k8s_openapi::api::{ + core::v1::ServiceAccount, + rbac::v1::{ClusterRole, ClusterRoleBinding, PolicyRule, Role, RoleBinding, RoleRef, Subject}, +}; +use kube::{ + Api, Client, + api::{DeleteParams, ListParams, Patch, PatchParams, PostParams, Preconditions}, +}; +use serde_json::json; + +const RETIRED: &str = "kars.azure.com/sre-legacy-retired"; +const GRANTS: &[(&str, &str)] = &[ + ("kars-sre-private-reader", "kars-sre-private-diagnostics"), + ("kars-sre-private-author", "kars-sre-action-author"), + ("kars-sre-private-renew", "kars-sre-router-renew"), +]; + +pub(crate) fn legacy_subject(subject: &Subject) -> bool { + (subject.kind == "ServiceAccount" + && subject.name == "sandbox" + && subject.namespace.as_deref() == Some(RUNTIME_NAMESPACE)) + || (subject.kind == "User" + && subject.name == format!("system:serviceaccount:{RUNTIME_NAMESPACE}:sandbox")) +} + +fn legacy_group(subject: &Subject) -> bool { + subject.kind == "Group" + && [ + "system:authenticated".to_string(), + "system:serviceaccounts".to_string(), + format!("system:serviceaccounts:{RUNTIME_NAMESPACE}"), + ] + .contains(&subject.name) +} + +fn dangerous(rules: &[PolicyRule]) -> bool { + rules.iter().any(|rule| { + let verbs = &rule.verbs; + let resources = rule.resources.as_deref().unwrap_or_default(); + let groups = rule.api_groups.as_deref().unwrap_or_default(); + let has = + |items: &[String], value: &str| items.iter().any(|item| item == "*" || item == value); + (has(groups, "") + && has(resources, "secrets") + && ["get", "list", "watch"].iter().any(|verb| has(verbs, verb))) + || (has(groups, "") + && ["pods/exec", "pods/proxy", "serviceaccounts/token"] + .iter() + .any(|resource| has(resources, resource)) + && (has(verbs, "create") || has(verbs, "get"))) + || (has(groups, "kars.azure.com") + && has(resources, "karssreactions") + && has(verbs, "create")) + }) +} + +async fn rules( + client: &Client, + role: &RoleRef, + namespace: Option<&str>, +) -> Result, String> { + if role.api_group != "rbac.authorization.k8s.io" { + return Err("Legacy binding roleRef uses an unsupported API group".into()); + } + match role.kind.as_str() { + "ClusterRole" => Ok(Api::::all(client.clone()) + .get(&role.name) + .await + .map_err(|e| api_error("Read legacy ClusterRole", e))? + .rules + .unwrap_or_default()), + "Role" => Ok(Api::::namespaced( + client.clone(), + namespace.ok_or("Role has no namespace")?, + ) + .get(&role.name) + .await + .map_err(|e| api_error("Read legacy Role", e))? + .rules + .unwrap_or_default()), + _ => Err("Legacy binding roleRef kind is invalid".into()), + } +} + +#[derive(Clone)] +pub(super) struct Binding { + review: BindingReview, + metadata: kube::api::ObjectMeta, + subjects: Vec, +} + +fn validate_review( + reg: &KarsSRERegistration, + kind: &str, + namespace: Option<&str>, + metadata: &kube::api::ObjectMeta, + role: &RoleRef, + subjects: &[Subject], +) -> Result { + let review = reg + .spec + .legacy_bindings + .iter() + .find(|review| { + review.kind == kind + && review.namespace.as_deref() == namespace + && Some(review.name.as_str()) == metadata.name.as_deref() + }) + .ok_or("Unreviewed legacy SRE binding exists; no migration mutations are authorized")?; + let retired = + metadata.annotations.as_ref().and_then(|a| a.get(RETIRED)) == reg.metadata.uid.as_ref(); + let expected: Vec<_> = review + .subjects + .iter() + .filter(|subject| !legacy_subject(subject)) + .cloned() + .collect(); + if metadata.uid.as_deref() != Some(review.uid.as_str()) + || metadata.deletion_timestamp.is_some() + || role != &review.role_ref + || (!retired + && (metadata.resource_version.as_deref() != Some(review.resource_version.as_str()) + || subjects != review.subjects)) + || (retired && subjects != expected) + { + return Err("Reviewed legacy grant changed or was replaced; migration stopped".into()); + } + Ok(Binding { + review: review.clone(), + metadata: metadata.clone(), + subjects: subjects.to_vec(), + }) +} + +pub(super) async fn review( + client: &Client, + reg: &KarsSRERegistration, +) -> Result, String> { + let cluster = Api::::all(client.clone()) + .list(&ListParams::default()) + .await + .map_err(|e| api_error("Inventory legacy ClusterRoleBindings", e))?; + let local = Api::::all(client.clone()) + .list(&ListParams::default()) + .await + .map_err(|e| api_error("Inventory legacy RoleBindings", e))?; + let mut bindings = Vec::new(); + let mut inspect = Vec::new(); + for binding in cluster { + inspect.push(( + "ClusterRoleBinding", + binding.metadata, + binding.role_ref, + binding.subjects.unwrap_or_default(), + )); + } + for binding in local { + inspect.push(( + "RoleBinding", + binding.metadata, + binding.role_ref, + binding.subjects.unwrap_or_default(), + )); + } + for (kind, metadata, role, subjects) in inspect { + if subjects.iter().any(legacy_group) + && dangerous(&rules(client, &role, metadata.namespace.as_deref()).await?) + { + return Err("A broad group grant gives legacy SRE credentials privileged access; restructure that grant explicitly".into()); + } + let was_reviewed = reg.spec.legacy_bindings.iter().any(|review| { + review.kind == kind + && review.name == metadata.name.as_deref().unwrap_or_default() + && review.namespace == metadata.namespace + }); + let legacy = subjects.iter().any(legacy_subject); + let ordinary_spawner = legacy + && role.kind == "ClusterRole" + && role.name == "kars-sandbox-spawner" + && rules(client, &role, metadata.namespace.as_deref()) + .await? + .iter() + .all(|rule| { + rule.api_groups.as_deref() == Some(&["kars.azure.com".into()][..]) + && rule.resources.as_deref() == Some(&["karssandboxes".into()][..]) + && rule.non_resource_urls.is_none() + && rule.verbs.iter().all(|verb| { + ["get", "list", "create", "delete"].contains(&verb.as_str()) + }) + }); + if (legacy && !ordinary_spawner) || was_reviewed { + bindings.push(validate_review( + reg, + kind, + metadata.namespace.as_deref(), + &metadata, + &role, + &subjects, + )?); + } + } + Ok(bindings) +} + +pub(super) async fn retire_legacy( + client: &Client, + reg: &KarsSRERegistration, + bindings: &[Binding], +) -> Result<(), String> { + for binding in bindings { + if !binding.subjects.iter().any(legacy_subject) { + continue; + } + let subjects: Vec<_> = binding + .subjects + .iter() + .filter(|subject| !legacy_subject(subject)) + .cloned() + .collect(); + let patch = json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version, + "annotations":{RETIRED:reg.metadata.uid}}, + "subjects":subjects, + }); + if binding.review.kind == "ClusterRoleBinding" { + Api::::all(client.clone()) + .patch( + &binding.review.name, + &PatchParams::default(), + &Patch::Merge(patch), + ) + .await + .map_err(|e| api_error("Retire reviewed SRE ClusterRoleBinding", e))?; + } else { + Api::::namespaced( + client.clone(), + binding.review.namespace.as_deref().unwrap(), + ) + .patch( + &binding.review.name, + &PatchParams::default(), + &Patch::Merge(patch), + ) + .await + .map_err(|e| api_error("Retire reviewed SRE RoleBinding", e))?; + } + } + Ok(()) +} + +fn owned(meta: &kube::api::ObjectMeta, reg: &KarsSRERegistration) -> bool { + meta.annotations.as_ref().is_some_and(|a| { + a.get(OWNER) == reg.metadata.uid.as_ref() + && a.get("kars.azure.com/namespace-uid") == Some(®.spec.runtime_namespace.uid) + }) && meta.uid.as_deref().is_some_and(|uid| !uid.is_empty()) + && meta + .resource_version + .as_deref() + .is_some_and(|rv| !rv.is_empty()) + && meta.deletion_timestamp.is_none() +} + +pub(super) async fn validate_private_targets( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let bindings: Api = Api::all(client.clone()); + for (name, role) in GRANTS { + let definition = Api::::all(client.clone()) + .get(role) + .await + .map_err(|e| api_error("Preflight private SRE role", e))?; + if !safe_private_role(role, definition.rules.as_deref().unwrap_or_default()) { + return Err( + "Private SRE role definition has excessive or unsupported authority".into(), + ); + } + if let Some(binding) = bindings + .get_opt(name) + .await + .map_err(|e| api_error("Preflight private SRE grant", e))? + && (!owned(&binding.metadata, reg) + || binding.role_ref.name != *role + || binding.role_ref.kind != "ClusterRole" + || binding.role_ref.api_group != "rbac.authorization.k8s.io" + || binding.subjects.as_deref().is_none_or(|subjects| { + subjects.len() != 1 + || subjects[0].kind != "ServiceAccount" + || subjects[0].name != ROUTER_SA + || subjects[0].namespace.as_deref() != Some(RUNTIME_NAMESPACE) + })) + { + return Err( + "Reserved private SRE binding already has another owner or authority".into(), + ); + } + } + let roles: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(role) = roles + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Preflight renewal Role", e))? + && (!owned(&role.metadata, reg) + || serde_json::to_value(&role.rules).ok() + != Some(json!([{ + "apiGroups":[""],"resources":["serviceaccounts/token"],"resourceNames":[ROUTER_SA],"verbs":["create"], + }]))) + { + return Err( + "Reserved renewal Role belongs to another owner or has different authority".into(), + ); + } + let bindings: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(binding) = bindings + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Preflight renewal binding", e))? + && (!owned(&binding.metadata, reg) + || binding.role_ref.kind != "Role" + || binding.role_ref.name != "sre-api-self-renew" + || binding.subjects.as_deref().is_none_or(|subjects| { + subjects.len() != 1 + || subjects[0].kind != "ServiceAccount" + || subjects[0].name != ROUTER_SA + || subjects[0].namespace.as_deref() != Some(RUNTIME_NAMESPACE) + })) + { + return Err( + "Reserved renewal binding belongs to another owner or has different authority".into(), + ); + } + Ok(()) +} + +pub(super) async fn grant_private( + client: &Client, + reg: &KarsSRERegistration, + sa: &ServiceAccount, +) -> Result<(), String> { + super::credential_guard::scan(client, reg).await?; + if !owned(&sa.metadata, reg) { + return Err("Private SRE ServiceAccount ownership is not proven".into()); + } + let subjects = vec![Subject { + kind: "ServiceAccount".into(), + name: ROUTER_SA.into(), + namespace: Some(RUNTIME_NAMESPACE.into()), + api_group: None, + }]; + let api: Api = Api::all(client.clone()); + for (name, role) in GRANTS { + let definition = Api::::all(client.clone()) + .get(role) + .await + .map_err(|e| api_error("Validate private SRE role definition", e))?; + if !safe_private_role(role, definition.rules.as_deref().unwrap_or_default()) { + return Err( + "Private SRE role definition contains unreviewed or excessive authority".into(), + ); + } + super::live::verify(client, reg).await?; + let body: ClusterRoleBinding = serde_json::from_value(json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":name,"annotations":{OWNER:reg.metadata.uid, + "kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":role}, + "subjects":subjects, + })) + .map_err(|_| "Private SRE binding serialization failed")?; + match api + .get_opt(name) + .await + .map_err(|e| api_error("Read private SRE binding", e))? + { + None => { + api.create(&PostParams::default(), &body) + .await + .map_err(|e| api_error("Create private SRE binding", e))?; + } + Some(existing) => { + if !owned(&existing.metadata, reg) + || existing.role_ref != body.role_ref + || existing.subjects != body.subjects + { + return Err("Private SRE binding conflicts with an existing resource".into()); + } + } + } + } + let roles: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let role: Role = serde_json::from_value(json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"Role", + "metadata":{"name":"sre-api-self-renew","namespace":RUNTIME_NAMESPACE,"annotations":{ + OWNER:reg.metadata.uid,"kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}}, + "rules":[{"apiGroups":[""],"resources":["serviceaccounts/token"],"resourceNames":[ROUTER_SA],"verbs":["create"]}], + })).map_err(|_| "SRE token renewal Role serialization failed")?; + match roles + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read renewal Role", e))? + { + None => { + roles + .create(&PostParams::default(), &role) + .await + .map_err(|e| api_error("Create renewal Role", e))?; + } + Some(existing) if owned(&existing.metadata, reg) && existing.rules == role.rules => {} + Some(_) => { + return Err( + "Existing SRE token renewal Role is not owned or has different authority".into(), + ); + } + } + let bindings: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let binding: RoleBinding = serde_json::from_value(json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBinding", + "metadata":{"name":"sre-api-self-renew","namespace":RUNTIME_NAMESPACE,"annotations":{ + OWNER:reg.metadata.uid,"kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":"sre-api-self-renew"}, + "subjects":subjects, + })).map_err(|_| "SRE renewal binding serialization failed")?; + match bindings + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read renewal binding", e))? + { + None => { + bindings + .create(&PostParams::default(), &binding) + .await + .map_err(|e| api_error("Create renewal binding", e))?; + } + Some(existing) + if owned(&existing.metadata, reg) + && existing.subjects == binding.subjects + && existing.role_ref == binding.role_ref => {} + Some(_) => { + return Err( + "Existing SRE renewal binding is not owned or has different authority".into(), + ); + } + } + Ok(()) +} + +fn safe_private_role(name: &str, rules: &[PolicyRule]) -> bool { + !rules.is_empty() + && rules.iter().all(|rule| { + let resources = rule.resources.as_deref().unwrap_or_default(); + let groups = rule.api_groups.as_deref().unwrap_or_default(); + if resources.is_empty() + || resources.iter().any(|r| { + r == "*" + || ["/proxy", "/exec", "/attach", "/portforward", "/token"] + .iter() + .any(|suffix| r.ends_with(suffix)) + }) + { + return false; + } + match name { + "kars-sre-private-diagnostics" => rule + .verbs + .iter() + .all(|verb| ["get", "list", "watch"].contains(&verb.as_str())), + "kars-sre-action-author" => { + groups == ["kars.azure.com"] + && resources.iter().all(|r| { + ["karssreactions", "karssreactions/status"].contains(&r.as_str()) + }) + && rule + .verbs + .iter() + .all(|verb| ["get", "list", "watch", "create"].contains(&verb.as_str())) + } + "kars-sre-router-renew" => { + (groups == ["kars.azure.com"] + && resources == ["karssreregistrations"] + && rule.resource_names.as_deref() == Some(&["canonical".into()][..]) + && rule + .verbs + .iter() + .all(|verb| ["get", "renew"].contains(&verb.as_str()))) + || (groups == ["authorization.k8s.io"] + && resources == ["subjectaccessreviews"] + && rule.verbs == ["create"] + && rule.non_resource_urls.is_none()) + } + _ => false, + } + }) +} + +pub(super) async fn revoke_private( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let private_subject = |subject: &&Subject| { + (subject.kind == "ServiceAccount" + && subject.name == ROUTER_SA + && subject.namespace.as_deref() == Some(RUNTIME_NAMESPACE)) + || (subject.kind == "User" + && subject.name == format!("system:serviceaccount:{RUNTIME_NAMESPACE}:{ROUTER_SA}")) + }; + let mut foreign = false; + let api: Api = Api::all(client.clone()); + for (name, _) in GRANTS { + if let Some(binding) = api + .get_opt(name) + .await + .map_err(|e| api_error("Read retiring SRE binding", e))? + { + if !owned(&binding.metadata, reg) { + foreign = true; + continue; + } + let retained: Vec<_> = binding + .subjects + .as_deref() + .unwrap_or_default() + .iter() + .filter(|subject| !private_subject(subject)) + .cloned() + .collect(); + if !retained.is_empty() { + api.patch(name, &PatchParams::default(), &Patch::Merge(json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version}, + "subjects":retained, + }))).await.map_err(|e| api_error("Revoke only the owned private SRE subject", e))?; + continue; + } + api.delete( + name, + &DeleteParams { + preconditions: Some(Preconditions { + uid: binding.metadata.uid, + resource_version: binding.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire private SRE binding", e))?; + } + } + let mut retain_role = false; + let bindings: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(binding) = bindings + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read retiring SRE renewal binding", e))? + { + if !owned(&binding.metadata, reg) { + return Err("Refusing to retire an unowned renewal binding".into()); + } + let retained: Vec<_> = binding + .subjects + .as_deref() + .unwrap_or_default() + .iter() + .filter(|subject| !private_subject(subject)) + .cloned() + .collect(); + if !retained.is_empty() { + bindings.patch("sre-api-self-renew", &PatchParams::default(), &Patch::Merge(json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version}, + "subjects":retained, + }))).await.map_err(|e| api_error("Revoke only the private SRE renewal subject", e))?; + retain_role = true; + } else { + bindings + .delete( + "sre-api-self-renew", + &DeleteParams { + preconditions: Some(Preconditions { + uid: binding.metadata.uid, + resource_version: binding.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire SRE renewal binding", e))?; + } + } + let roles: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(role) = roles + .get_opt("sre-api-self-renew") + .await + .map_err(|e| api_error("Read retiring SRE renewal Role", e))? + && !retain_role + { + if !owned(&role.metadata, reg) { + return Err("Refusing to retire an unowned renewal Role".into()); + } + roles + .delete( + "sre-api-self-renew", + &DeleteParams { + preconditions: Some(Preconditions { + uid: role.metadata.uid, + resource_version: role.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire SRE renewal Role", e))?; + } + if foreign { + Err("Unowned private SRE bindings were preserved; owned grants were revoked".into()) + } else { + Ok(()) + } +} diff --git a/controller/src/sre_authority/credential_guard.rs b/controller/src/sre_authority/credential_guard.rs new file mode 100644 index 000000000..93d23d904 --- /dev/null +++ b/controller/src/sre_authority/credential_guard.rs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::api_error; +use crate::sre_registration::{KarsSRERegistration, ROUTER_SA, RUNTIME_NAMESPACE}; +use k8s_openapi::api::core::v1::{Secret, ServiceAccount}; +use kube::{Api, Client, api::ListParams}; + +pub(super) async fn scan(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + let account = Api::::namespaced(client.clone(), RUNTIME_NAMESPACE) + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Inspect reserved SRE token identity", e))?; + let mut uids = Vec::new(); + if let Some(uid) = account.as_ref().and_then(|sa| sa.metadata.uid.as_deref()) { + uids.push(uid); + } + if let Some(uid) = reg + .status + .as_ref() + .and_then(|status| status.router_service_account_uid.as_deref()) + { + uids.push(uid); + } + let metadata = Api::::namespaced(client.clone(), RUNTIME_NAMESPACE) + .list_metadata(&ListParams::default()) + .await + .map_err(|e| api_error("Inspect prestaged SRE token Secret metadata", e))?; + let list = serde_json::to_value(metadata) + .map_err(|_| "SRE token metadata inventory could not be decoded")?; + crate::sre_privacy::reject_legacy_aliases(&list, &uids).map_err(str::to_owned) +} diff --git a/controller/src/sre_authority/credentials.rs b/controller/src/sre_authority/credentials.rs new file mode 100644 index 000000000..3213c71e5 --- /dev/null +++ b/controller/src/sre_authority/credentials.rs @@ -0,0 +1,423 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::{api_error, live::Authority}; +use crate::sre_registration::{ + AGENT_SECRET, EPOCH, KarsSRERegistration, OWNER, PRIVATE_SECRET, ROUTER_SA, RUNTIME_NAMESPACE, +}; +use k8s_openapi::api::{ + authentication::v1::TokenRequest, + core::v1::{Secret, ServiceAccount}, +}; +use kube::{ + Api, Client, ResourceExt, + api::{DeleteParams, Patch, PatchParams, PostParams, Preconditions}, +}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +fn annotations(reg: &KarsSRERegistration) -> Value { + json!({OWNER:reg.metadata.uid,EPOCH:reg.epoch(), + "kars.azure.com/sandbox-uid":reg.spec.sandbox.uid, + "kars.azure.com/namespace-uid":reg.spec.runtime_namespace.uid}) +} + +fn owned(meta: &kube::api::ObjectMeta, reg: &KarsSRERegistration) -> bool { + meta.namespace.as_deref() == Some(RUNTIME_NAMESPACE) + && meta.annotations.as_ref().is_some_and(|a| { + a.get(OWNER) == reg.metadata.uid.as_ref() + && a.get("kars.azure.com/sandbox-uid") == Some(®.spec.sandbox.uid) + && a.get("kars.azure.com/namespace-uid") == Some(®.spec.runtime_namespace.uid) + }) + && meta.uid.as_deref().is_some_and(|uid| !uid.is_empty()) + && meta + .resource_version + .as_deref() + .is_some_and(|rv| !rv.is_empty()) + && meta.deletion_timestamp.is_none() +} + +fn data(secret: &Secret, key: &str) -> Option { + String::from_utf8(secret.data.as_ref()?.get(key)?.0.clone()).ok() +} + +pub(super) async fn review_targets( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + super::credential_guard::scan(client, reg).await?; + let accounts: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(sa) = accounts + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Preflight private SRE identity", e))? + && (!owned(&sa.metadata, reg) || sa.automount_service_account_token != Some(false)) + { + return Err("Reserved SRE ServiceAccount already belongs to another owner".into()); + } + let secrets: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + for name in [PRIVATE_SECRET, AGENT_SECRET] { + if let Some(secret) = secrets + .get_metadata_opt(name) + .await + .map_err(|e| api_error("Preflight SRE material ownership", e))? + && !owned(&secret.metadata, reg) + { + return Err("Reserved SRE identity Secret already belongs to another owner".into()); + } + } + Ok(()) +} + +pub(super) async fn ensure_service_account( + client: &Client, + reg: &KarsSRERegistration, + authority: &Authority, +) -> Result { + super::credential_guard::scan(client, reg).await?; + if authority.namespace.metadata.uid.as_deref() != Some(®.spec.runtime_namespace.uid) + || authority.sandbox.metadata.uid.as_deref() != Some(®.spec.sandbox.uid) + { + return Err("SRE authority changed before private identity creation".into()); + } + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(sa) = api + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Read private SRE ServiceAccount", e))? + { + if !owned(&sa.metadata, reg) + || sa.automount_service_account_token != Some(false) + || reg + .status + .as_ref() + .and_then(|s| s.router_service_account_uid.as_deref()) + .is_some_and(|uid| Some(uid) != sa.metadata.uid.as_deref()) + { + return Err("Private SRE ServiceAccount is unowned, replaced, or unsafe".into()); + } + return Ok(sa); + } + let sa: ServiceAccount = serde_json::from_value(json!({ + "apiVersion":"v1","kind":"ServiceAccount","metadata":{ + "name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE,"annotations":annotations(reg)}, + "automountServiceAccountToken":false, + })) + .map_err(|_| "Private SRE ServiceAccount serialization failed")?; + api.create(&PostParams::default(), &sa) + .await + .map_err(|e| api_error("Create private SRE ServiceAccount", e)) +} + +async fn secret(client: &Client, reg: &KarsSRERegistration, name: &str) -> Result { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(secret) = api + .get_opt(name) + .await + .map_err(|e| api_error("Read owned SRE identity material", e))? + { + if !owned(&secret.metadata, reg) { + return Err( + "Existing SRE identity material has a different owner or namespace incarnation" + .into(), + ); + } + return Ok(secret); + } + let empty: Secret = serde_json::from_value(json!({ + "apiVersion":"v1","kind":"Secret","type":"Opaque", + "metadata":{"name":name,"namespace":RUNTIME_NAMESPACE,"annotations":annotations(reg)}, + })) + .map_err(|_| "SRE identity metadata serialization failed")?; + api.create(&PostParams::default(), &empty) + .await + .map_err(|e| api_error("Create owned SRE identity metadata", e)) +} + +async fn update( + client: &Client, + secret: &Secret, + values: BTreeMap, + annotations: Value, +) -> Result<(), String> { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + api.patch(&secret.name_any(),&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":secret.metadata.uid,"resourceVersion":secret.metadata.resource_version,"annotations":annotations}, + "type":"Opaque","stringData":values, + }))).await.map_err(|e|api_error("Update owned SRE identity material",e))?; + Ok(()) +} + +fn cluster_connection() -> Result<(String, String), String> { + let host = std::env::var("KUBERNETES_SERVICE_HOST") + .map_err(|_| "Kubernetes service host is unavailable")?; + let port = std::env::var("KUBERNETES_SERVICE_PORT_HTTPS").unwrap_or_else(|_| "443".into()); + let host = if host.contains(':') { + format!("[{host}]") + } else { + host + }; + let url = reqwest::Url::parse(&format!("https://{host}:{port}")) + .map_err(|_| "Kubernetes service address is invalid")?; + let ca = std::fs::read_to_string("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt") + .map_err(|_| "Kubernetes service CA is unavailable")?; + Ok((url.to_string(), ca)) +} + +pub(super) async fn ensure( + client: &Client, + reg: &KarsSRERegistration, + authority: &Authority, + sa: &ServiceAccount, +) -> Result<(), String> { + ensure_with_connection(client, reg, authority, sa, cluster_connection).await +} + +async fn ensure_with_connection( + client: &Client, + reg: &KarsSRERegistration, + _authority: &Authority, + sa: &ServiceAccount, + connection: impl FnOnce() -> Result<(String, String), String>, +) -> Result<(), String> { + super::live::verify(client, reg).await?; + super::check_secret_denial(client, RUNTIME_NAMESPACE).await?; + super::credential_guard::scan(client, reg).await?; + let mut private = secret(client, reg, PRIVATE_SECRET).await?; + if private + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(EPOCH)) + != Some(®.epoch()) + || (reg + .status + .as_ref() + .is_some_and(|status| status.phase == "Blocked") + && data(&private, "kube-token").is_some()) + { + // Replacing the bound Secret UID invalidates old private JWTs, not just + // the current file contents, when upgrading an underguarded epoch. + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + api.delete( + PRIVATE_SECRET, + &DeleteParams { + preconditions: Some(Preconditions { + uid: private.metadata.uid.clone(), + resource_version: private.metadata.resource_version.clone(), + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire prior-epoch SRE token anchor", e))?; + private = secret(client, reg, PRIVATE_SECRET).await?; + } + let agent = secret(client, reg, AGENT_SECRET).await?; + let mut private_values = BTreeMap::new(); + let now = chrono::Utc::now().timestamp(); + let epoch = reg.epoch(); + let tls_expiry = private + .metadata + .annotations + .as_ref() + .and_then(|a| a.get("kars.azure.com/sre-tls-expiry")) + .and_then(|value| value.parse::().ok()) + .unwrap_or_default(); + let same_epoch = private + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(EPOCH)) + == Some(&epoch); + let tls_valid = same_epoch + && tls_expiry > now + 172_800 + && [ + "server-cert.pem", + "server-key.pem", + "agent-token", + "agent-ca.crt", + ] + .iter() + .all(|key| data(&private, key).is_some()); + let mut private_annotations = annotations(reg); + if !tls_valid { + let identity = crate::providers::sre_tls::issue()?; + let proxy_token = crate::providers::signing::generate_service_token(); + private_values.insert("server-cert.pem".into(), identity.certificate); + private_values.insert("server-key.pem".into(), identity.private_key); + private_values.insert("agent-ca.crt".into(), identity.ca.clone()); + private_values.insert("agent-token".into(), proxy_token.clone()); + private_annotations["kars.azure.com/sre-tls-expiry"] = + identity.expires_at.to_string().into(); + // Token and CA change together. The legacy Hermes client rebuilds its + // TLS context when it observes the opaque token change. + update( + client, + &agent, + BTreeMap::from([ + ("token".into(), proxy_token), + ("ca.crt".into(), identity.ca), + ("namespace".into(), RUNTIME_NAMESPACE.into()), + ]), + annotations(reg), + ) + .await?; + } else if data(&agent, "token") != data(&private, "agent-token") + || data(&agent, "ca.crt") != data(&private, "agent-ca.crt") + { + update( + client, + &agent, + BTreeMap::from([ + ( + "token".into(), + data(&private, "agent-token").ok_or("SRE proxy token missing")?, + ), + ( + "ca.crt".into(), + data(&private, "agent-ca.crt").ok_or("SRE proxy CA missing")?, + ), + ("namespace".into(), RUNTIME_NAMESPACE.into()), + ]), + annotations(reg), + ) + .await?; + } + let kube_expiry = data(&private, "kube-expires-at") + .and_then(|value| chrono::DateTime::parse_from_rfc3339(&value).ok()) + .map(|time| time.timestamp()) + .unwrap_or_default(); + if !same_epoch || kube_expiry < now + 600 { + let request: TokenRequest = serde_json::from_value(json!({ + "apiVersion":"authentication.k8s.io/v1","kind":"TokenRequest", + "spec":{"audiences":[],"expirationSeconds":3600, + "boundObjectRef":{"apiVersion":"v1","kind":"Secret","name":PRIVATE_SECRET,"uid":private.metadata.uid}}, + })).map_err(|_|"SRE TokenRequest serialization failed")?; + let token = Api::::namespaced(client.clone(), RUNTIME_NAMESPACE) + .create_token_request(ROUTER_SA, &PostParams::default(), &request) + .await + .map_err(|e| api_error("Issue private SRE Kubernetes token", e))?; + let token = token + .status + .ok_or("Kubernetes TokenRequest omitted its token/expiry")?; + let expiry = token.expiration_timestamp.0.to_string(); + if token.token.is_empty() + || chrono::DateTime::parse_from_rfc3339(&expiry) + .map_err(|_| "TokenRequest expiry was invalid")? + .timestamp() + <= now + 300 + { + return Err("TokenRequest returned unusable private credentials".into()); + } + let (url, ca) = connection()?; + private_values.insert("kube-token".into(), token.token); + private_values.insert("kube-expires-at".into(), expiry); + private_values.insert("kube-ca.crt".into(), ca); + private_values.insert( + "config.json".into(), + serde_json::to_string(&json!({ + "schema":"kars.azure.com/sre-api/v1","kubeUrl":url, + "registrationUid":reg.metadata.uid,"privacyEpoch":epoch, + "source":reg.spec.sandbox,"namespaceUid":reg.spec.runtime_namespace.uid, + "runtimeNamespace":RUNTIME_NAMESPACE,"serviceAccountUid":sa.metadata.uid, + "secretUid":private.metadata.uid, + })) + .map_err(|_| "SRE proxy configuration serialization failed")?, + ); + } + + if !private_values.is_empty() { + super::live::verify(client, reg).await?; + super::check_secret_denial(client, RUNTIME_NAMESPACE).await?; + super::credential_guard::scan(client, reg).await?; + update(client, &private, private_values, private_annotations).await?; + } + Ok(()) +} + +#[cfg(test)] +pub(super) async fn ensure_for_test( + client: &Client, + reg: &KarsSRERegistration, + authority: &Authority, + sa: &ServiceAccount, +) -> Result<(), String> { + ensure_with_connection(client, reg, authority, sa, || { + Ok(("https://kubernetes.default.svc".into(), "test-ca".into())) + }) + .await +} + +pub(super) async fn retire(client: &Client, reg: &KarsSRERegistration) -> Result<(), String> { + let secrets: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + for name in [PRIVATE_SECRET, AGENT_SECRET] { + if let Some(secret) = secrets + .get_opt(name) + .await + .map_err(|e| api_error("Read retiring SRE identity", e))? + { + if secret + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(OWNER)) + != reg.metadata.uid.as_ref() + { + continue; + } + if !owned(&secret.metadata, reg) { + return Err("SRE identity material has conflicting incarnation metadata".into()); + } + secrets + .delete( + name, + &DeleteParams { + preconditions: Some(Preconditions { + uid: secret.metadata.uid, + resource_version: secret.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire owned SRE identity", e))?; + } + } + let accounts: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(sa) = accounts + .get_opt(ROUTER_SA) + .await + .map_err(|e| api_error("Read retiring private SRE identity", e))? + { + if sa.metadata.annotations.as_ref().and_then(|a| a.get(OWNER)) != reg.metadata.uid.as_ref() + { + return Ok(()); + } + if !owned(&sa.metadata, reg) { + return Err("Private SRE identity has conflicting incarnation metadata".into()); + } + if reg + .status + .as_ref() + .and_then(|status| status.router_service_account_uid.as_deref()) + .is_some_and(|uid| sa.metadata.uid.as_deref() != Some(uid)) + { + return Err("Private SRE ServiceAccount UID changed; replacement preserved".into()); + } + accounts + .delete( + ROUTER_SA, + &DeleteParams { + preconditions: Some(Preconditions { + uid: sa.metadata.uid, + resource_version: sa.metadata.resource_version, + }), + ..Default::default() + }, + ) + .await + .map_err(|e| api_error("Retire private SRE identity", e))?; + } + Ok(()) +} diff --git a/controller/src/sre_authority/live.rs b/controller/src/sre_authority/live.rs new file mode 100644 index 000000000..62a20bf36 --- /dev/null +++ b/controller/src/sre_authority/live.rs @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + crd::KarsSandbox, + sre_registration::{KarsSRERegistration, NAME}, +}; +use k8s_openapi::api::{ + apps::v1::Deployment, authorization::v1::SubjectAccessReview, core::v1::Namespace, +}; +use kube::{Api, Client, api::PostParams}; + +pub(super) struct Authority { + pub sandbox: KarsSandbox, + pub namespace: Namespace, +} + +pub(crate) fn api_error(stage: &str, error: kube::Error) -> String { + match error { + kube::Error::Api(status) => format!("{stage}: Kubernetes status {}", status.code), + _ => format!("{stage}: Kubernetes transport/serialization failure"), + } +} + +pub(super) async fn verify( + client: &Client, + reg: &KarsSRERegistration, +) -> Result { + reg.validate()?; + let registrations: Api = Api::all(client.clone()); + let current = registrations + .get(NAME) + .await + .map_err(|e| api_error("Read live SRE registration", e))?; + if current.metadata.uid != reg.metadata.uid + || current.metadata.generation != reg.metadata.generation + || current.metadata.deletion_timestamp.is_some() + { + return Err("SRE registration changed during reconciliation".into()); + } + let namespaces: Api = Api::all(client.clone()); + let control_ns = namespaces + .get(®.spec.controller.namespace.name) + .await + .map_err(|e| api_error("Read registered controller namespace", e))?; + if control_ns.metadata.uid.as_deref() != Some(®.spec.controller.namespace.uid) + || control_ns.metadata.deletion_timestamp.is_some() + { + return Err("Registered controller namespace was replaced or is terminating".into()); + } + let deployments: Api = + Api::namespaced(client.clone(), ®.spec.controller.namespace.name); + let controller = deployments + .get(®.spec.controller.deployment.name) + .await + .map_err(|e| api_error("Read registered controller Deployment", e))?; + if controller.metadata.uid.as_deref() != Some(®.spec.controller.deployment.uid) + || controller.metadata.deletion_timestamp.is_some() + || controller + .metadata + .annotations + .as_ref() + .is_some_and(|annotations| { + annotations + .get("meta.helm.sh/release-name") + .is_some_and(|name| name != ®.spec.controller.release) + || annotations + .get("meta.helm.sh/release-namespace") + .is_some_and(|name| name != ®.spec.controller.namespace.name) + }) + { + return Err("Registered controller/release identity does not match".into()); + } + let sandboxes: Api = Api::namespaced(client.clone(), ®.spec.sandbox.namespace); + let sandbox = sandboxes + .get(®.spec.sandbox.name) + .await + .map_err(|e| api_error("Read registered SRE Sandbox", e))?; + let namespace = namespaces + .get(®.spec.runtime_namespace.name) + .await + .map_err(|e| api_error("Read registered SRE runtime namespace", e))?; + if sandbox.metadata.uid.as_deref() != Some(®.spec.sandbox.uid) + || sandbox.metadata.deletion_timestamp.is_some() + || sandbox + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get("kars.azure.com/role")) + .map(String::as_str) + != Some("sre") + || namespace.metadata.uid.as_deref() != Some(®.spec.runtime_namespace.uid) + || namespace.metadata.deletion_timestamp.is_some() + || sandbox + .metadata + .annotations + .as_ref() + .and_then(|a| a.get("kars.azure.com/namespace-uid")) + != Some(®.spec.runtime_namespace.uid) + || !crate::reconciler::namespace_ownership::claimed(&namespace, &sandbox) + .map_err(|_| "Registered SRE namespace claim is invalid")? + { + return Err( + "Registered SRE source/runtime UID or claim does not match; foreign occupant preserved" + .into(), + ); + } + Ok(Authority { sandbox, namespace }) +} + +pub(crate) async fn check_secret_denial(client: &Client, namespace: &str) -> Result<(), String> { + let reviews: Api = Api::all(client.clone()); + for request in crate::sre_privacy::secret_access_reviews(namespace) { + let review: SubjectAccessReview = serde_json::from_value(request) + .map_err(|_| "SRE Secret authorization request is invalid")?; + let checked = reviews + .create(&PostParams::default(), &review) + .await + .map_err(|e| api_error("Verify legacy SRE Secret denial", e))?; + let response = serde_json::to_value(checked) + .map_err(|_| "SRE Secret authorization response is invalid")?; + crate::sre_privacy::require_denial(&response)?; + } + Ok(()) +} + +/// Later private-credential issuers must call this immediately before issuance. +/// An absent registration is safe only when the old SRE subject has no access. +pub(crate) async fn privacy_epoch( + client: &Client, + target_namespace: &str, +) -> Result, String> { + check_secret_denial(client, target_namespace).await?; + let registrations: Api = Api::all(client.clone()); + let Some(reg) = registrations + .get_opt(NAME) + .await + .map_err(|e| api_error("Read SRE privacy epoch", e))? + else { + return Ok(None); + }; + if !reg.spec.enabled + && reg.status.as_ref().is_some_and(|status| { + status.phase == "Retired" + && status.observed_generation == reg.metadata.generation.unwrap_or_default() + }) + { + return Ok(None); + } + verify(client, ®).await?; + super::admission::verify(client).await?; + super::credential_guard::scan(client, ®).await?; + let status = reg + .status + .as_ref() + .ok_or("SRE authority has not been reconciled")?; + if status.phase != "Ready" + || status.observed_generation != reg.metadata.generation.unwrap_or_default() + || !status.legacy_secret_access_denied + || status.privacy_revision.as_deref() != Some(crate::sre_privacy::REVISION) + || status.privacy_epoch.as_deref() != Some(reg.epoch().as_str()) + { + return Err("SRE privacy migration is not Ready".into()); + } + Ok(status.privacy_epoch.clone()) +} diff --git a/controller/src/sre_authority/migration.rs b/controller/src/sre_authority/migration.rs new file mode 100644 index 000000000..a0f9cbe4d --- /dev/null +++ b/controller/src/sre_authority/migration.rs @@ -0,0 +1,353 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::{api_error, check_secret_denial}; +use crate::{ + crd::KarsSandbox, + sre_registration::{EPOCH, KarsSRERegistration, OWNER, RUNTIME_NAMESPACE}, +}; +use k8s_openapi::api::{ + apps::v1::Deployment, + core::v1::{Namespace, Pod, Secret}, +}; +use kube::{ + Api, Client, ResourceExt, + api::{ListParams, Patch, PatchParams}, +}; +use serde_json::json; + +const STOPPED: &str = "kars.azure.com/sre-migration-stopped"; +pub(super) const WAITING_FOR_CONSUMERS: &str = + "Waiting for legacy SRE consumers to terminate; no private credentials issued"; +const WAITING_FOR_ROTATION: &str = + "Waiting for owned control credential consumers to restart on the new privacy epoch"; +const WAITING_FOR_ROLLOUT: &str = + "Owned control credential consumer has not completed its privacy-epoch rollout"; + +pub(super) fn is_waiting(detail: &str) -> bool { + matches!( + detail, + WAITING_FOR_CONSUMERS | WAITING_FOR_ROTATION | WAITING_FOR_ROLLOUT + ) +} + +pub(super) async fn stop_registered_consumer_for_retirement( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let namespaces: Api = Api::all(client.clone()); + let Some(namespace) = namespaces + .get_opt(RUNTIME_NAMESPACE) + .await + .map_err(|e| api_error("Read retiring SRE namespace", e))? + else { + return Ok(()); + }; + if namespace.metadata.uid.as_deref() != Some(reg.spec.runtime_namespace.uid.as_str()) { + return Ok(()); + } + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let Some(deployment) = api + .get_opt("sre") + .await + .map_err(|e| api_error("Read retiring SRE consumer", e))? + else { + return Ok(()); + }; + let ours = deployment + .spec + .as_ref() + .and_then(|s| s.template.metadata.as_ref()) + .and_then(|m| m.annotations.as_ref()) + .and_then(|a| a.get(OWNER)) + == reg.metadata.uid.as_ref(); + if !ours + && reg + .status + .as_ref() + .and_then(|s| s.router_service_account_uid.as_ref()) + .is_none() + { + return Ok(()); + } + stop_legacy_consumer(client, reg).await +} + +fn current_boundary(deployment: &Deployment, reg: &KarsSRERegistration) -> bool { + deployment.spec.as_ref().is_some_and(|spec| { + spec.template + .metadata + .as_ref() + .and_then(|m| m.annotations.as_ref()) + .is_some_and(|a| { + a.get(OWNER) == reg.metadata.uid.as_ref() && a.get(EPOCH) == Some(®.epoch()) + }) + }) +} + +pub(super) async fn validate_consumer( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let Some(deployment) = api + .get_opt("sre") + .await + .map_err(|e| api_error("Read SRE consumer", e))? + else { + return Ok(()); + }; + if current_boundary(&deployment, reg) { + return Ok(()); + } + if deployment + .spec + .as_ref() + .and_then(|s| s.template.metadata.as_ref()) + .and_then(|m| m.annotations.as_ref()) + .and_then(|a| a.get(OWNER)) + == reg.metadata.uid.as_ref() + { + return Ok(()); + } + let reviewed = reg + .spec + .legacy_consumer + .as_ref() + .ok_or("Existing SRE consumer requires an explicit UID/resourceVersion review")?; + let was_stopped = deployment + .metadata + .annotations + .as_ref() + .and_then(|a| a.get(STOPPED)) + == reg.metadata.uid.as_ref(); + if deployment.metadata.uid.as_deref() != Some(reviewed.uid.as_str()) + || deployment.metadata.deletion_timestamp.is_some() + || (!was_stopped + && deployment.metadata.resource_version.as_deref() + != Some(reviewed.resource_version.as_str())) + { + return Err("Reviewed SRE consumer was changed or replaced".into()); + } + Ok(()) +} + +pub(super) async fn stop_legacy_consumer( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + if let Some(deployment) = api + .get_opt("sre") + .await + .map_err(|e| api_error("Read SRE consumer for retirement", e))? + { + if reg.spec.enabled && current_boundary(&deployment, reg) { + return Ok(()); + } + validate_consumer(client, reg).await?; + if deployment.spec.as_ref().and_then(|spec| spec.replicas) != Some(0) { + api.patch("sre",&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":deployment.metadata.uid,"resourceVersion":deployment.metadata.resource_version, + "annotations":{STOPPED:reg.metadata.uid}}, + "spec":{"replicas":0}, + }))).await.map_err(|e|api_error("Stop reviewed SRE consumer",e))?; + } + } + let pods: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); + let pods = pods + .list(&ListParams::default()) + .await + .map_err(|e| api_error("Wait for old SRE consumers", e))?; + if pods.iter().any(|pod| { + pod.spec + .as_ref() + .and_then(|spec| spec.service_account_name.as_deref()) + == Some("sandbox") + }) { + return Err(WAITING_FOR_CONSUMERS.into()); + } + Ok(()) +} + +fn controller_managed(deployment: &Deployment, sandbox: &str) -> bool { + deployment + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get("kars.azure.com/sandbox")) + .map(String::as_str) + == Some(sandbox) + && deployment + .metadata + .managed_fields + .as_ref() + .is_some_and(|fields| { + fields.iter().any(|field| { + field.manager.as_deref() == Some(crate::field_managers::CLAWSANDBOX) + && field + .fields_v1 + .as_ref() + .is_some_and(|fields| fields.0.get("f:spec").is_some()) + }) + }) +} + +/// Existing #550 control credentials may have been exposed by the old SRE +/// identity. Rotate only proven controller-owned instances and restart their +/// owned consumer; the old router caches the token at startup. +pub(super) async fn rotate_owned_control_credentials( + client: &Client, + reg: &KarsSRERegistration, +) -> Result<(), String> { + let all: Api = Api::all(client.clone()); + let list = all + .list(&ListParams::default().fields("metadata.name=router-services-admin")) + .await + .map_err(|e| api_error("Inventory owned control credentials", e))?; + for secret in list { + if secret + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get("app.kubernetes.io/managed-by")) + .map(String::as_str) + != Some("kars-controller") + { + continue; + } + if secret.metadata.uid.as_deref().is_none_or(str::is_empty) + || secret + .metadata + .resource_version + .as_deref() + .is_none_or(str::is_empty) + || secret.metadata.deletion_timestamp.is_some() + || secret + .metadata + .owner_references + .as_ref() + .is_some_and(|owners| !owners.is_empty()) + { + return Err("Owned control credential identity is incomplete or terminating".into()); + } + let Some(namespace_name) = secret.namespace() else { + continue; + }; + let annotations = secret.metadata.annotations.as_ref(); + let Some(source_uid) = annotations.and_then(|a| a.get("kars.azure.com/sandbox-uid")) else { + continue; + }; + let Some(namespace_uid) = annotations.and_then(|a| a.get("kars.azure.com/namespace-uid")) + else { + continue; + }; + let namespaces: Api = Api::all(client.clone()); + let namespace = namespaces + .get(&namespace_name) + .await + .map_err(|e| api_error("Read control credential namespace", e))?; + if namespace.metadata.uid.as_deref() != Some(namespace_uid) + || namespace.metadata.deletion_timestamp.is_some() + { + return Err("Control credential namespace changed".into()); + } + let ns_annotations = namespace + .metadata + .annotations + .as_ref() + .ok_or("Control namespace claim is absent")?; + let workspace = ns_annotations + .get("kars.azure.com/sandbox-namespace") + .ok_or("Control namespace source is absent")?; + let name = ns_annotations + .get("kars.azure.com/sandbox-name") + .ok_or("Control namespace owner is absent")?; + let sandboxes: Api = Api::namespaced(client.clone(), workspace); + let sandbox = sandboxes + .get(name) + .await + .map_err(|e| api_error("Read control credential owner", e))?; + if sandbox.metadata.uid.as_deref() != Some(source_uid) + || sandbox.metadata.deletion_timestamp.is_some() + || !crate::reconciler::namespace_ownership::claimed(&namespace, &sandbox) + .map_err(|_| "Control namespace ownership is invalid")? + { + return Err("Control credential ownership changed; no rotation was authorized".into()); + } + check_secret_denial(client, &namespace_name).await?; + let deployments: Api = Api::namespaced(client.clone(), &namespace_name); + let Some(deployment) = deployments + .get_opt(name) + .await + .map_err(|e| api_error("Read control credential consumer", e))? + else { + return Err("Owned control credential has no verified consumer Deployment".into()); + }; + if !controller_managed(&deployment, name) { + return Err("Control credential consumer is not controller-owned".into()); + } + let epoch = reg.epoch(); + if annotations.and_then(|a| a.get(EPOCH)) != Some(&epoch) { + let secrets: Api = Api::namespaced(client.clone(), &namespace_name); + secrets.patch("router-services-admin",&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":secret.metadata.uid,"resourceVersion":secret.metadata.resource_version, + "annotations":{EPOCH:epoch}}, + "stringData":{"control-token":crate::providers::signing::generate_service_token()}, + }))).await.map_err(|e|api_error("Rotate owned control credential",e))?; + } + if deployment + .spec + .as_ref() + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|meta| meta.annotations.as_ref()) + .and_then(|a| a.get(EPOCH)) + != Some(&epoch) + { + deployments.patch(name,&PatchParams::default(),&Patch::Merge(json!({ + "metadata":{"uid":deployment.metadata.uid,"resourceVersion":deployment.metadata.resource_version}, + "spec":{"template":{"metadata":{"annotations":{EPOCH:epoch}}}}, + }))).await.map_err(|e|api_error("Restart owned control credential consumer",e))?; + return Err(WAITING_FOR_ROTATION.into()); + } + let desired = deployment + .spec + .as_ref() + .and_then(|spec| spec.replicas) + .unwrap_or(1); + let ready = deployment.status.as_ref().is_some_and(|status| { + status.observed_generation == deployment.metadata.generation + && status.updated_replicas.unwrap_or(0) == desired + && status.available_replicas.unwrap_or(0) == desired + }); + if !ready { + return Err(WAITING_FOR_ROLLOUT.into()); + } + let selector = deployment + .spec + .as_ref() + .and_then(|spec| spec.selector.match_labels.as_ref()) + .filter(|labels| !labels.is_empty()) + .ok_or("Owned control credential consumer has no bounded Pod selector")? + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join(","); + let pods = Api::::namespaced(client.clone(), &namespace_name) + .list(&ListParams::default().labels(&selector)) + .await + .map_err(|e| api_error("Verify old control credential consumers have terminated", e))?; + if pods.iter().any(|pod| { + pod.metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(EPOCH)) + != Some(&epoch) + }) { + // Rollout availability alone can exclude a still-terminating old + // router which continues accepting its startup-cached control token. + return Err(WAITING_FOR_ROLLOUT.into()); + } + } + Ok(()) +} diff --git a/controller/src/sre_authority/pod.rs b/controller/src/sre_authority/pod.rs new file mode 100644 index 000000000..0067508b9 --- /dev/null +++ b/controller/src/sre_authority/pod.rs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + crd::KarsSandbox, + sre_registration::{AGENT_SECRET, EPOCH, KarsSRERegistration, NAME, OWNER, PRIVATE_SECRET}, +}; +use k8s_openapi::api::core::v1::{Namespace, Secret}; +use kube::{Api, Client, ResourceExt}; +use serde_json::{Value, json}; + +pub(crate) struct Projection { + pub epoch: String, + pub registration_uid: String, + pub tls_expiry: String, + pub credential_uid: String, +} + +pub(crate) async fn authorize( + client: &Client, + sandbox: &KarsSandbox, + namespace: &Namespace, +) -> Result, String> { + let reserved = sandbox.name_any() == "sre" + || sandbox + .metadata + .labels + .as_ref() + .and_then(|l| l.get("kars.azure.com/role")) + .map(String::as_str) + == Some("sre"); + if !reserved { + return Ok(None); + } + let registrations: Api = Api::all(client.clone()); + let reg = registrations + .get_opt(NAME) + .await + .map_err(|e| super::api_error("Read SRE enrollment", e))? + .ok_or("SRE source is staged but not enrolled by a cluster registrar")?; + if reg.spec.sandbox.uid != sandbox.uid().unwrap_or_default() + || reg.spec.sandbox.namespace != sandbox.namespace().unwrap_or_default() + || reg.spec.runtime_namespace.uid != namespace.uid().unwrap_or_default() + { + return Err("This SRE occupant is not the registered canonical source; no private identity is granted".into()); + } + let epoch = super::privacy_epoch(client, ®.spec.runtime_namespace.name) + .await? + .ok_or("SRE privacy epoch is absent")?; + if !reg.spec.enabled + || reg.status.as_ref().is_none_or(|status| { + status.phase != "Ready" + || status.observed_generation != reg.metadata.generation.unwrap_or_default() + || status.privacy_epoch.as_deref() != Some(epoch.as_str()) + || !status.legacy_secret_access_denied + }) + { + return Err("SRE authority migration has not reached Ready".into()); + } + let secrets: Api = Api::namespaced(client.clone(), ®.spec.runtime_namespace.name); + let secret = secrets + .get_metadata(PRIVATE_SECRET) + .await + .map_err(|e| super::api_error("Read SRE TLS revision", e))?; + let annotations = secret + .metadata + .annotations + .as_ref() + .ok_or("SRE private identity annotations missing")?; + if annotations.get(OWNER) != reg.metadata.uid.as_ref() || annotations.get(EPOCH) != Some(&epoch) + { + return Err("SRE private identity does not match the registered privacy epoch".into()); + } + Ok(Some(Projection { + epoch, + registration_uid: reg.uid().ok_or("SRE registration UID missing")?, + credential_uid: secret.uid().ok_or("SRE private credential UID missing")?, + tls_expiry: annotations + .get("kars.azure.com/sre-tls-expiry") + .cloned() + .ok_or("SRE TLS expiry missing")?, + })) +} + +fn env(container: &mut Value, name: &str, value: &str) { + let env = container["env"] + .as_array_mut() + .expect("controller container environment"); + env.retain(|entry| entry["name"] != name); + env.push(json!({"name":name,"value":value})); +} + +pub(crate) fn project(pod: &mut Value) { + // Do not change serviceAccountName: Azure federation remains + // system:serviceaccount::sandbox. Only the K8s diagnostic client + // uses the separate private identity. + pod["automountServiceAccountToken"] = false.into(); + let volumes = pod["volumes"] + .as_array_mut() + .expect("controller pod volumes"); + volumes.extend([ + json!({"name":"sre-api-private","secret":{"secretName":PRIVATE_SECRET}}), + json!({"name":"sre-api-agent","secret":{"secretName":AGENT_SECRET, + "items":[{"key":"token","path":"token"},{"key":"ca.crt","path":"ca.crt"},{"key":"namespace","path":"namespace"}]}}), + json!({"name":"router-kubernetes","projected":{"sources":[ + {"serviceAccountToken":{"path":"token","expirationSeconds":3600}}, + {"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}}, + {"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"fieldPath":"metadata.namespace"}}]}} + ]}}), + ]); + for container in pod["containers"] + .as_array_mut() + .expect("controller pod containers") + { + let router = container["name"] == "inference-router"; + container["volumeMounts"] + .as_array_mut() + .expect("controller mounts") + .push(json!({ + "name":if router {"router-kubernetes"} else {"sre-api-agent"}, + "mountPath":"/var/run/secrets/kubernetes.io/serviceaccount","readOnly":true, + })); + if router { + container["volumeMounts"] + .as_array_mut() + .unwrap() + .push(json!({ + "name":"sre-api-private","mountPath":"/etc/kars/sre-api","readOnly":true, + })); + env(container, "KARS_SRE_API_ENABLED", "true"); + container["readinessProbe"] = json!({ + "exec":{"command":["kars-inference-router","sre-ready"]}, + "initialDelaySeconds":3,"periodSeconds":5,"timeoutSeconds":5, + }); + } else { + let prior = container["env"] + .as_array() + .and_then(|values| values.iter().find(|entry| entry["name"] == "NO_PROXY")) + .and_then(|entry| entry["value"].as_str()) + .unwrap_or_default(); + let no_proxy = format!("127.0.0.1,localhost,{prior}"); + env(container, "KUBERNETES_SERVICE_HOST", "127.0.0.1"); + env(container, "KUBERNETES_SERVICE_PORT", "9446"); + env(container, "KUBERNETES_SERVICE_PORT_HTTPS", "9446"); + env(container, "NO_PROXY", &no_proxy); + env(container, "no_proxy", &no_proxy); + } + } +} + +pub(crate) fn annotations(projection: &Projection, agent_name: &str) -> Value { + json!({ + OWNER:projection.registration_uid,EPOCH:projection.epoch, + "kars.azure.com/sre-tls-expiry":projection.tls_expiry, + "kars.azure.com/sre-private-credential-uid":projection.credential_uid, + "azure.workload.identity/skip-containers":format!("{agent_name},egress-guard"), + }) +} diff --git a/controller/src/sre_authority/privacy_tests.rs b/controller/src/sre_authority/privacy_tests.rs new file mode 100644 index 000000000..f52d2b003 --- /dev/null +++ b/controller/src/sre_authority/privacy_tests.rs @@ -0,0 +1,424 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::tests::{State, fixture, registration}; +use super::*; +use crate::sre_registration::*; +use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; + +const REG: &str = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical"; +const SECRETS: &str = "/api/v1/namespaces/kars-sre/secrets"; +const SA: &str = "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router"; +const CRBS: &str = "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings"; + +fn ready() -> KarsSRERegistration { + let mut reg = registration(); + reg.status = Some( + serde_json::from_value(json!({ + "phase":"Ready","observedGeneration":1,"privacyEpoch":reg.epoch(), + "routerServiceAccountUid":"router-sa","legacySecretAccessDenied":true, + "privacyRevision":crate::sre_privacy::REVISION, + })) + .unwrap(), + ); + reg +} + +fn admission_ready(state: &Arc>) { + let mut state = state.lock().unwrap(); + for name in admission::POLICIES { + state.objects.insert(format!("/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/{name}"),json!({ + "metadata":{"name":name,"generation":1},"spec":{"failurePolicy":"Fail","validations":[]}, + "status":{"observedGeneration":1,"typeChecking":{}}})); + state.objects.insert( + format!( + "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings/{name}" + ), + json!({ + "metadata":{"name":name},"spec":{"policyName":name,"validationActions":["Deny"]}}), + ); + } +} + +fn alias(name: &str, account_name: &str, account_uid: Option<&str>) -> Value { + let mut secret = json!({"apiVersion":"v1","kind":"Secret","type":"kubernetes.io/service-account-token", + "metadata":{"name":name,"namespace":"kars-sre","uid":format!("uid-{name}"),"resourceVersion":"1", + "annotations":{"kubernetes.io/service-account.name":account_name}}, + "data":{"token":"PRIVATE_TOKEN_SENTINEL"}}); + if let Some(uid) = account_uid { + secret["metadata"]["annotations"]["kubernetes.io/service-account.uid"] = uid.into(); + } + secret +} + +#[tokio::test] +async fn prestaged_aliases_and_recreated_uids_never_receive_a_service_account_or_credentials() { + for (name, uid) in [ + ("sre-api-router", None), + ("sre-api-router", Some("obsolete-sa")), + ("renamed-before-policy", Some("router-sa")), + ] { + let (_server, client, state) = fixture().await; + let reg = ready(); + state.lock().unwrap().objects.insert( + format!("{SECRETS}/arbitrary-alias"), + alias("arbitrary-alias", name, uid), + ); + let authority = live::verify(&client, ®).await.unwrap(); + let error = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap_err(); + assert!(!error.contains("PRIVATE_TOKEN_SENTINEL")); + let state = state.lock().unwrap(); + assert!(state.calls.iter().all(|(method, _, _)| method == "GET")); + assert!( + state + .objects + .contains_key(&format!("{SECRETS}/arbitrary-alias")) + ); + assert!( + state + .metadata_requests + .iter() + .all(|accept| accept.contains("PartialObjectMetadataList")) + ); + } +} + +#[tokio::test] +async fn unsafe_alias_after_ready_revokes_owned_grants_but_preserves_foreign_and_unrelated_subjects() + { + let (_server, client, state) = fixture().await; + let reg = ready(); + admission_ready(&state); + { + let mut state = state.lock().unwrap(); + state + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + state.objects.insert( + format!("{SECRETS}/alias"), + alias("alias", "sre-api-router", Some("router-sa")), + ); + state.objects.insert( + SA.into(), + json!({"metadata":{"name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE, + "uid":"router-sa","resourceVersion":"1"}}), + ); + for (name, role, owner) in [ + ( + "kars-sre-private-reader", + "kars-sre-private-diagnostics", + "registration", + ), + ( + "kars-sre-private-author", + "kars-sre-action-author", + "foreign", + ), + ( + "kars-sre-private-renew", + "kars-sre-router-renew", + "registration", + ), + ] { + let mut binding = json!({"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":name,"uid":format!("uid-{name}"),"resourceVersion":"1", + "annotations":{OWNER:owner,"kars.azure.com/namespace-uid":"runtime-ns"}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":role}, + "subjects":[{"kind":"ServiceAccount","name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE}]}); + if name == "kars-sre-private-renew" { + binding["subjects"].as_array_mut().unwrap().push(json!({"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"})); + } + state.objects.insert(format!("{CRBS}/{name}"), binding); + } + } + assert!(reconcile(&client, ®).await.is_err()); + let state = state.lock().unwrap(); + assert!(state.objects.contains_key(&format!("{SECRETS}/alias"))); + assert!( + !state + .objects + .contains_key(&format!("{CRBS}/kars-sre-private-reader")) + ); + assert!( + state + .objects + .contains_key(&format!("{CRBS}/kars-sre-private-author")) + ); + assert_eq!( + state.objects[&format!("{CRBS}/kars-sre-private-renew")]["subjects"], + json!([{"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]) + ); + assert_eq!(state.objects[REG]["status"]["phase"], "Blocked"); + assert_eq!( + state.objects[REG]["status"]["legacySecretAccessDenied"], + false + ); + let patch = &state + .calls + .iter() + .find(|(_, path, _)| path.ends_with("/status")) + .unwrap() + .2; + assert!(patch["status"]["privacyEpoch"].is_null()); + assert!(patch["status"]["privacyRevision"].is_null()); + assert!(state.calls.iter().all(|(method, path, _)| method == "GET" + || path.starts_with(CRBS) + || path.ends_with("/status"))); +} + +#[tokio::test] +async fn watch_only_and_wildcard_group_grants_fail_review_before_mutation() { + for rule in [ + json!({"apiGroups":[""],"resources":["secrets"],"verbs":["watch"]}), + json!({"apiGroups":["*"],"resources":["*"],"verbs":["watch"]}), + json!({"apiGroups":[""],"resources":["secrets"],"verbs":["*"]}), + json!({"apiGroups":[""],"resources":["pods","secrets"],"verbs":["get","watch"]}), + ] { + let (_server, client, state) = fixture().await; + { + let mut state = state.lock().unwrap(); + state.binding["subjects"] = json!([{"kind":"Group","name":"system:serviceaccounts:kars-sre", + "apiGroup":"rbac.authorization.k8s.io"}]); + state.objects.insert( + "/apis/rbac.authorization.k8s.io/v1/clusterroles/kars-sre-reader".into(), + json!({"metadata":{"name":"kars-sre-reader"},"rules":[rule]}), + ); + } + assert!( + bindings::review(&client, ®istration()) + .await + .err() + .unwrap() + .contains("broad group") + ); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); + } +} + +#[tokio::test] +async fn live_reviews_cover_namespace_and_cluster_watch_including_named_grants() { + for (namespace, name) in [ + (Some("kars-sre"), None), + (None, None), + (Some("kars-sre"), Some(PRIVATE_SECRET)), + (None, Some("router-services-admin")), + ] { + let (_server, client, state) = fixture().await; + state.lock().unwrap().watch_allowed = + Some((namespace.map(str::to_owned), name.map(str::to_owned))); + assert!(check_secret_denial(&client, "kars-sre").await.is_err()); + assert!(privacy_epoch(&client, "kars-sre").await.is_err()); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .any(|(_, _, request)| request["spec"]["resourceAttributes"]["verb"] == "watch") + ); + } +} + +#[tokio::test] +async fn inventory_api_errors_and_prior_get_list_only_status_are_not_privacy_safe() { + let (_server, client, state) = fixture().await; + admission_ready(&state); + let mut reg = ready(); + reg.status.as_mut().unwrap().privacy_revision = None; + state + .lock() + .unwrap() + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + assert!(privacy_epoch(&client, RUNTIME_NAMESPACE).await.is_err()); + state.lock().unwrap().api_error = Some(403); + let error = credential_guard::scan(&client, ®).await.unwrap_err(); + assert!(!error.contains("PRIVATE_SENTINEL")); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, path, _)| method == "GET" || path.ends_with("/subjectaccessreviews")) + ); +} + +#[tokio::test] +async fn blocked_recovery_replaces_the_owned_token_anchor_uid_without_recreating_pod_identity() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let path = format!("{SECRETS}/{PRIVATE_SECRET}"); + let old_uid = state.lock().unwrap().objects[&path]["metadata"]["uid"].clone(); + reg.status = ready().status; + reg.status.as_mut().unwrap().phase = "Blocked".into(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let state = state.lock().unwrap(); + assert_ne!(state.objects[&path]["metadata"]["uid"], old_uid); + assert_eq!( + state + .calls + .iter() + .filter(|(method, path, _)| method == "DELETE" && path.ends_with(PRIVATE_SECRET)) + .count(), + 1 + ); + assert!( + !state + .calls + .iter() + .any(|(method, path, _)| method == "DELETE" && path.contains("/serviceaccounts/")) + ); +} + +#[tokio::test] +async fn newly_discovered_watch_group_revokes_previously_ready_owned_reader() { + let (_server, client, state) = fixture().await; + let reg = ready(); + admission_ready(&state); + { + let mut state = state.lock().unwrap(); + state + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + state.binding["subjects"] = json!([{"kind":"Group","name":"system:serviceaccounts:kars-sre", + "apiGroup":"rbac.authorization.k8s.io"}]); + state.objects.insert("/apis/rbac.authorization.k8s.io/v1/clusterroles/kars-sre-reader".into(), + json!({"metadata":{"name":"kars-sre-reader"},"rules":[{"apiGroups":[""],"resources":["secrets"],"verbs":["watch"]}]})); + state.objects.insert(format!("{CRBS}/kars-sre-private-reader"),json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":"kars-sre-private-reader","uid":"owned-reader","resourceVersion":"1", + "annotations":{OWNER:"registration","kars.azure.com/namespace-uid":"runtime-ns"}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"kars-sre-private-diagnostics"}, + "subjects":[{"kind":"ServiceAccount","name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE}]})); + } + assert!(reconcile(&client, ®).await.is_err()); + let state = state.lock().unwrap(); + assert!( + !state + .objects + .contains_key(&format!("{CRBS}/kars-sre-private-reader")) + ); + assert_eq!(state.objects[REG]["status"]["phase"], "Blocked"); + assert!(state.calls.iter().all(|(method, path, _)| method == "GET" + || path == &format!("{CRBS}/kars-sre-private-reader") + || path.ends_with("/status"))); +} + +#[tokio::test] +async fn annotation_changes_and_secret_recreation_are_rescanned_and_preserved() { + let (_server, client, state) = fixture().await; + let reg = ready(); + let path = format!("{SECRETS}/alias"); + state + .lock() + .unwrap() + .objects + .insert(path.clone(), alias("alias", "ordinary", None)); + credential_guard::scan(&client, ®).await.unwrap(); + for uid in ["original", "recreated"] { + { + let mut state = state.lock().unwrap(); + state.objects.get_mut(&path).unwrap()["metadata"]["uid"] = uid.into(); + state.objects.get_mut(&path).unwrap()["metadata"]["annotations"]["kubernetes.io/service-account.name"] = + ROUTER_SA.into(); + } + assert!(credential_guard::scan(&client, ®).await.is_err()); + assert_eq!(state.lock().unwrap().objects[&path]["metadata"]["uid"], uid); + } + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn quarantine_retires_owned_identity_but_preserves_the_unsafe_unknown_alias() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + reg.status = ready().status; + reg.status.as_mut().unwrap().router_service_account_uid = sa.metadata.uid.clone(); + admission_ready(&state); + { + let mut state = state.lock().unwrap(); + state + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + state.objects.insert( + format!("{SECRETS}/unsafe-alias"), + alias("unsafe-alias", ROUTER_SA, sa.metadata.uid.as_deref()), + ); + state.calls.clear(); + } + assert!(reconcile(&client, ®).await.is_err()); + let state = state.lock().unwrap(); + assert!( + state + .objects + .contains_key(&format!("{SECRETS}/unsafe-alias")) + ); + assert!(!state.objects.contains_key(SA)); + assert!( + !state + .objects + .contains_key(&format!("{SECRETS}/{PRIVATE_SECRET}")) + ); + assert!(state.calls.iter().all(|(method, path, _)| { + method != "DELETE" + || [ + SA, + &format!("{SECRETS}/{PRIVATE_SECRET}"), + &format!("{SECRETS}/{AGENT_SECRET}"), + ] + .contains(&path.as_str()) + })); +} + +#[tokio::test] +async fn private_service_account_replacement_uid_is_never_deleted_during_quarantine() { + let (_server, client, state) = fixture().await; + let reg = ready(); + state.lock().unwrap().objects.insert(SA.into(),json!({ + "metadata":{"name":ROUTER_SA,"namespace":RUNTIME_NAMESPACE,"uid":"replacement","resourceVersion":"2", + "annotations":{OWNER:"registration","kars.azure.com/sandbox-uid":"source","kars.azure.com/namespace-uid":"runtime-ns"}}})); + assert!(credentials::retire(&client, ®).await.is_err()); + assert!(state.lock().unwrap().objects.contains_key(SA)); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} diff --git a/controller/src/sre_authority/tests.rs b/controller/src/sre_authority/tests.rs new file mode 100644 index 000000000..b6d2a7519 --- /dev/null +++ b/controller/src/sre_authority/tests.rs @@ -0,0 +1,564 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use crate::sre_registration::*; +use base64::Engine; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +pub(super) fn registration() -> KarsSRERegistration { + serde_json::from_value(json!({ + "apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSRERegistration", + "metadata":{"name":"canonical","uid":"registration","resourceVersion":"1","generation":1}, + "spec":{"controller":{"namespace":{"name":"kars-system","uid":"control-ns"}, + "deployment":{"name":"kars-controller","uid":"controller"},"release":"kars"}, + "sandbox":{"namespace":"kars-system","name":"sre","uid":"source"}, + "runtimeNamespace":{"name":"kars-sre","uid":"runtime-ns"},"enabled":true, + "legacyBindings":[{"kind":"ClusterRoleBinding","name":"legacy","uid":"binding","resourceVersion":"1", + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"kars-sre-reader"}, + "subjects":[{"kind":"ServiceAccount","name":"sandbox","namespace":"kars-sre"}, + {"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]}]}, + })).unwrap() +} + +fn sandbox() -> Value { + json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSandbox", + "metadata":{"name":"sre","namespace":"kars-system","uid":"source","resourceVersion":"1", + "labels":{"kars.azure.com/role":"sre"},"annotations":{"kars.azure.com/namespace-uid":"runtime-ns"}}, + "spec":{"runtime":{"kind":"Hermes","hermes":{}},"inferenceRef":{"name":"sre-inference"}}}) +} + +fn namespace() -> Value { + json!({"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kars-sre","uid":"runtime-ns","resourceVersion":"1", + "annotations":{"kars.azure.com/namespace-claim-version":"v1","kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre","kars.azure.com/sandbox-uid":"source"}}}) +} + +fn binding() -> Value { + json!({"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBinding", + "metadata":{"name":"legacy","uid":"binding","resourceVersion":"1"}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"kars-sre-reader"}, + "subjects":[{"kind":"ServiceAccount","name":"sandbox","namespace":"kars-sre"}, + {"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]}) +} + +fn api_error(code: u16) -> ResponseTemplate { + ResponseTemplate::new(code).set_body_json(json!({"apiVersion":"v1","kind":"Status","status":"Failure", + "code":code,"reason":if code==404 {"NotFound"} else {"Forbidden"},"message":"PRIVATE_SENTINEL"})) +} + +pub(super) struct State { + pub(super) sandbox: Value, + pub(super) namespace: Value, + pub(super) binding: Value, + pub(super) allowed: bool, + pub(super) calls: Vec<(String, String, Value)>, + pub(super) api_error: Option, + pub(super) objects: BTreeMap, + pub(super) watch_allowed: Option<(Option, Option)>, + pub(super) metadata_requests: Vec, +} + +pub(super) async fn fixture() -> (MockServer, Client, Arc>) { + let state = Arc::new(Mutex::new(State { + sandbox: sandbox(), + namespace: namespace(), + binding: binding(), + allowed: false, + calls: Vec::new(), + api_error: None, + objects: BTreeMap::new(), + watch_allowed: None, + metadata_requests: Vec::new(), + })); + let server = MockServer::start().await; + let handler = state.clone(); + Mock::given(|_:&wiremock::Request|true).respond_with(move |request:&wiremock::Request| { + let mut state=handler.lock().unwrap(); + let path=request.url.path(); + let body:Value=request.body_json().unwrap_or(Value::Null); + state.calls.push((request.method.to_string(),path.into(),body.clone())); + if request.method == "GET" && path=="/api/v1/namespaces/kars-sre/secrets" { + state.metadata_requests.push(request.headers.get("accept").unwrap().to_str().unwrap().into()); + } + if let Some(code)=state.api_error {return api_error(code)} + if request.method == "GET" && let Some(value) = state.objects.get(path) { + return ResponseTemplate::new(200).set_body_json(value); + } + if request.method == "DELETE" && let Some(value) = state.objects.get(path) { + if body["preconditions"]["uid"] != value["metadata"]["uid"] || + body["preconditions"]["resourceVersion"] != value["metadata"]["resourceVersion"] { return api_error(409); } + state.objects.remove(path); + return ResponseTemplate::new(200).set_body_json(json!({"kind":"Status","apiVersion":"v1","status":"Success"})); + } + if request.method == "POST" && (path.ends_with("/secrets") || path.ends_with("/serviceaccounts")) { + let mut value = body.clone(); + value["metadata"]["uid"] = format!("uid-{}-{}", body["metadata"]["name"].as_str().unwrap(),state.calls.len()).into(); + value["metadata"]["resourceVersion"] = "1".into(); + state.objects.insert(format!("{path}/{}",body["metadata"]["name"].as_str().unwrap()),value.clone()); + return ResponseTemplate::new(201).set_body_json(value); + } + if request.method == "PATCH" && path.contains("/secrets/") { + let Some(value) = state.objects.get_mut(path) else { return api_error(404); }; + if body["metadata"]["uid"] != value["metadata"]["uid"] || + body["metadata"]["resourceVersion"] != value["metadata"]["resourceVersion"] { return api_error(409); } + for (key, annotation) in body["metadata"]["annotations"].as_object().unwrap() { + value["metadata"]["annotations"][key] = annotation.clone(); + } + for (key, material) in body["stringData"].as_object().unwrap() { + value["data"][key] = base64::engine::general_purpose::STANDARD.encode(material.as_str().unwrap()).into(); + } + value["metadata"]["resourceVersion"] = + (value["metadata"]["resourceVersion"].as_str().unwrap().parse::().unwrap()+1).to_string().into(); + return ResponseTemplate::new(200).set_body_json(value.clone()); + } + if request.method == "PATCH" && body["subjects"].is_array() && state.objects.contains_key(path) { + let value=state.objects.get_mut(path).unwrap(); + if body["metadata"]["uid"]!=value["metadata"]["uid"] || + body["metadata"]["resourceVersion"]!=value["metadata"]["resourceVersion"] {return api_error(409);} + value["subjects"]=body["subjects"].clone(); + return ResponseTemplate::new(200).set_body_json(value.clone()); + } + let value=match (request.method.as_str(),path) { + ("GET","/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical")=>serde_json::to_value(registration()).unwrap(), + ("GET","/api/v1/namespaces/kars-system")=>json!({"metadata":{"name":"kars-system","uid":"control-ns","resourceVersion":"1"}}), + ("GET","/apis/apps/v1/namespaces/kars-system/deployments/kars-controller")=>json!({ + "metadata":{"name":"kars-controller","namespace":"kars-system","uid":"controller","resourceVersion":"1", + "annotations":{"meta.helm.sh/release-name":"kars","meta.helm.sh/release-namespace":"kars-system"}}}), + ("GET","/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/sre")=>state.sandbox.clone(), + ("GET","/api/v1/namespaces/kars-sre")=>state.namespace.clone(), + ("GET","/api/v1/namespaces/kars-sre/secrets")=>json!({ + "apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{}, + "items":state.objects.iter().filter(|(path,_)|path.starts_with("/api/v1/namespaces/kars-sre/secrets/")) + .map(|(_,object)|json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":object["metadata"]})) + .collect::>()}), + ("GET","/apis/rbac.authorization.k8s.io/v1/clusterrolebindings")=>json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBindingList","metadata":{},"items":[state.binding]}), + ("GET","/apis/rbac.authorization.k8s.io/v1/rolebindings")=>json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBindingList","metadata":{},"items":[]}), + ("PATCH","/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/legacy")=>{ + if body["metadata"]["uid"]!=state.binding["metadata"]["uid"] + || body["metadata"]["resourceVersion"]!=state.binding["metadata"]["resourceVersion"] {return api_error(409)} + state.binding["subjects"]=body["subjects"].clone(); + state.binding["metadata"]["annotations"]=body["metadata"]["annotations"].clone(); + state.binding.clone() + } + ("POST","/apis/authorization.k8s.io/v1/subjectaccessreviews")=>json!({ + "apiVersion":"authorization.k8s.io/v1","kind":"SubjectAccessReview", + "spec":body["spec"],"status":{"allowed":state.allowed || state.watch_allowed.as_ref().is_some_and(|(namespace,name)| + body["spec"]["resourceAttributes"]["verb"]=="watch" + && body["spec"]["resourceAttributes"]["namespace"].as_str()==namespace.as_deref() + && body["spec"]["resourceAttributes"]["name"].as_str()==name.as_deref())}}), + ("PATCH","/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical/status")=>{ + let mut reg=state.objects.get("/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical") + .cloned().unwrap_or_else(||serde_json::to_value(registration()).unwrap()); + reg["status"]=body["status"].clone(); + reg["metadata"]["resourceVersion"]="2".into(); + state.objects.insert("/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical".into(),reg.clone()); + reg + } + ("POST","/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token")=>json!({ + "apiVersion":"authentication.k8s.io/v1","kind":"TokenRequest","metadata":{},"spec":body["spec"], + "status":{"token":"PRIVATE_KUBE_TOKEN","expirationTimestamp":(chrono::Utc::now()+chrono::Duration::hours(1)).to_rfc3339()}}), + _=>return api_error(404), + }; + ResponseTemplate::new(200).set_body_json(value) + }).mount(&server).await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let client = Client::try_from(kube::Config::new(server.uri().parse().unwrap())).unwrap(); + (server, client, state) +} + +#[tokio::test] +async fn canonical_source_and_namespace_uids_authorize_not_labels_or_names() { + let (_server, client, state) = fixture().await; + let reg = registration(); + assert!(live::verify(&client, ®).await.is_ok()); + for target in ["source", "namespace", "foreign-workspace", "controller"] { + let mut reg = registration(); + match target { + "source" => reg.spec.sandbox.uid = "recreated".into(), + "namespace" => reg.spec.runtime_namespace.uid = "recreated".into(), + "controller" => reg.spec.controller.deployment.uid = "recreated".into(), + _ => { + state.lock().unwrap().namespace["metadata"]["annotations"]["kars.azure.com/sandbox-namespace"] = + "untrusted".into() + } + } + assert!(live::verify(&client, ®).await.is_err(), "{target}"); + } + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn reviewed_retirement_preserves_other_subjects_and_has_cas_preconditions() { + let (_server, client, state) = fixture().await; + let reg = registration(); + let reviewed = bindings::review(&client, ®).await.unwrap(); + bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap(); + let state = state.lock().unwrap(); + assert_eq!( + state.binding["subjects"], + json!([{"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]) + ); + let patch = &state + .calls + .iter() + .find(|(method, _, _)| method == "PATCH") + .unwrap() + .2; + assert_eq!(patch["metadata"]["uid"], "binding"); + assert_eq!(patch["metadata"]["resourceVersion"], "1"); +} + +#[tokio::test] +async fn unreviewed_or_replaced_bindings_stop_before_mutation() { + let (_server, client, state) = fixture().await; + for changed in ["missing-review", "uid", "version"] { + let mut reg = registration(); + match changed { + "missing-review" => reg.spec.legacy_bindings.clear(), + "uid" => reg.spec.legacy_bindings[0].uid = "other".into(), + _ => reg.spec.legacy_bindings[0].resource_version = "other".into(), + } + assert!(bindings::review(&client, ®).await.is_err()); + } + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn real_authorization_review_denial_is_required_and_errors_do_not_hide_access() { + let (_server, client, state) = fixture().await; + check_secret_denial(&client, "kars-sre").await.unwrap(); + assert_eq!( + state + .lock() + .unwrap() + .calls + .iter() + .filter(|(method, _, _)| method == "POST") + .count(), + 18 + ); + state.lock().unwrap().allowed = true; + assert!(check_secret_denial(&client, "kars-sre").await.is_err()); + state.lock().unwrap().api_error = Some(403); + let error = check_secret_denial(&client, "kars-sre").await.unwrap_err(); + assert!(!error.contains("PRIVATE_SENTINEL")); +} + +#[test] +fn proxy_projection_preserves_azure_identity_and_pinned_agent_image() { + let projection = pod::Projection { + epoch: "epoch".into(), + registration_uid: "registration".into(), + tls_expiry: "expiry".into(), + credential_uid: "credential-uid".into(), + }; + let mut spec = json!({"serviceAccountName":"sandbox","volumes":[],"containers":[ + {"name":"agent","image":"customer/hermes:pinned","env":[],"volumeMounts":[]}, + {"name":"inference-router","env":[],"volumeMounts":[]} ]}); + pod::project(&mut spec); + assert_eq!(spec["serviceAccountName"], "sandbox"); + assert_eq!(spec["containers"][0]["image"], "customer/hermes:pinned"); + assert_eq!(spec["automountServiceAccountToken"], false); + let agent = serde_json::to_string(&spec["containers"][0]).unwrap(); + assert!(!agent.contains(PRIVATE_SECRET)); + assert!(!agent.contains("router-kubernetes")); + assert!(agent.contains("sre-api-agent")); + assert!(agent.contains("127.0.0.1")); + assert!(agent.contains("9446")); + assert_eq!( + pod::annotations(&projection, "agent")["azure.workload.identity/skip-containers"], + "agent,egress-guard" + ); +} + +#[test] +fn registration_is_cluster_scoped_and_requires_complete_exact_identity() { + use kube::CustomResourceExt; + let crd = KarsSRERegistration::crd(); + assert_eq!(crd.spec.scope, "Cluster"); + let mut reg = registration(); + reg.validate().unwrap(); + reg.spec.sandbox.namespace = "untrusted".into(); + assert!(reg.validate().is_err()); + reg = registration(); + reg.metadata.uid = None; + assert!(reg.validate().is_err()); +} + +#[tokio::test] +async fn admission_requires_observed_enforced_policies_without_type_warnings() { + let (_server, client, state) = fixture().await; + let path = "/apis/admissionregistration.k8s.io/v1"; + for name in admission::POLICIES { + state.lock().unwrap().objects.insert(format!("{path}/validatingadmissionpolicies/{name}"),json!({ + "metadata":{"name":name,"generation":2},"spec":{"failurePolicy":"Fail","validations":[]}, + "status":{"observedGeneration":2,"typeChecking":{}}})); + state.lock().unwrap().objects.insert(format!("{path}/validatingadmissionpolicybindings/{name}"),json!({ + "metadata":{"name":name},"spec":{"policyName":name,"validationActions":["Deny","Audit"]}})); + } + admission::verify(&client).await.unwrap(); + let policy = format!( + "{path}/validatingadmissionpolicies/{}", + admission::POLICIES[0] + ); + state.lock().unwrap().objects.get_mut(&policy).unwrap()["status"]["typeChecking"] = json!({"expressionWarnings":[{"fieldRef":"spec.validations[0].expression","warning":"undeclared field"}]}); + assert!(admission::verify(&client).await.is_err()); + state.lock().unwrap().objects.get_mut(&policy).unwrap()["status"] = + json!({"observedGeneration":1,"typeChecking":{}}); + assert!(admission::verify(&client).await.is_err()); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); +} + +#[tokio::test] +async fn credentials_are_private_bound_and_renewable_without_agent_jwt() { + let (_server, client, state) = fixture().await; + let reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let private_path = "/api/v1/namespaces/kars-sre/secrets/sre-api-router-identity"; + let agent_path = "/api/v1/namespaces/kars-sre/secrets/sre-api-agent"; + let decode = |value: &Value, key: &str| { + String::from_utf8( + base64::engine::general_purpose::STANDARD + .decode(value["data"][key].as_str().unwrap()) + .unwrap(), + ) + .unwrap() + }; + let before = { + let locked = state.lock().unwrap(); + let private = &locked.objects[private_path]; + let agent = &locked.objects[agent_path]; + assert_eq!(decode(private, "kube-token"), "PRIVATE_KUBE_TOKEN"); + assert_eq!(decode(agent, "token"), decode(private, "agent-token")); + assert_eq!(decode(agent, "ca.crt"), decode(private, "agent-ca.crt")); + assert_eq!(agent["data"].as_object().unwrap().len(), 3); + assert!(!serde_json::to_string(agent).unwrap().contains("kube-token")); + let request = locked + .calls + .iter() + .find(|(_, path, _)| path.ends_with("/token")) + .unwrap(); + assert_eq!(request.2["spec"]["boundObjectRef"]["kind"], "Secret"); + assert_eq!( + request.2["spec"]["boundObjectRef"]["uid"], + private["metadata"]["uid"] + ); + assert!(decode(private, "server-key.pem").starts_with("-----BEGIN PRIVATE KEY-----")); + private.clone() + }; + let writes = state + .lock() + .unwrap() + .calls + .iter() + .filter(|(method, _, _)| method == "PATCH") + .count(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + assert_eq!( + writes, + state + .lock() + .unwrap() + .calls + .iter() + .filter(|(method, _, _)| method == "PATCH") + .count() + ); + state.lock().unwrap().objects.get_mut(private_path).unwrap()["metadata"]["annotations"]["kars.azure.com/sre-tls-expiry"] = + "0".into(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + let locked = state.lock().unwrap(); + assert_ne!( + decode(&before, "agent-token"), + decode(&locked.objects[private_path], "agent-token") + ); + assert_eq!( + decode(&locked.objects[private_path], "agent-token"), + decode(&locked.objects[agent_path], "token") + ); +} + +#[tokio::test] +async fn unsafe_authorization_or_recreated_targets_never_receive_private_material() { + for problem in [ + "secret-access", + "source-recreated", + "namespace-recreated", + "foreign-secret", + "api-error", + ] { + let (_server, client, state) = fixture().await; + let reg = registration(); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + { + let mut locked = state.lock().unwrap(); + locked.calls.clear(); + match problem { + "secret-access" => locked.allowed = true, + "source-recreated" => locked.sandbox["metadata"]["uid"] = "other".into(), + "namespace-recreated" => locked.namespace["metadata"]["uid"] = "other".into(), + "foreign-secret" => { + locked.objects.insert("/api/v1/namespaces/kars-sre/secrets/sre-api-router-identity".into(), + json!({"apiVersion":"v1","kind":"Secret","metadata":{"name":"sre-api-router-identity","namespace":"kars-sre","uid":"foreign","resourceVersion":"1"}})); + } + _ => locked.api_error = Some(403), + } + } + let error = credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap_err(); + assert!(!error.contains("PRIVATE_SENTINEL")); + assert!(state.lock().unwrap().calls.iter().all(|(method,path,_)| + method=="GET" || path.ends_with("/subjectaccessreviews")), "{problem}"); + } +} + +#[tokio::test] +async fn migration_waits_for_consumers_and_retires_only_owned_private_material() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.legacy_consumer = Some(ConsumerReview { + namespace: RUNTIME_NAMESPACE.into(), + name: "sre".into(), + uid: "consumer".into(), + resource_version: "1".into(), + }); + state.lock().unwrap().objects.insert("/apis/apps/v1/namespaces/kars-sre/deployments/sre".into(), + json!({"metadata":{"name":"sre","namespace":"kars-sre","uid":"consumer","resourceVersion":"1"}, + "spec":{"replicas":0,"selector":{"matchLabels":{"app":"sre"}},"template":{"metadata":{},"spec":{"containers":[]}}}})); + state.lock().unwrap().objects.insert( + "/api/v1/namespaces/kars-sre/pods".into(), + json!({"kind":"PodList","apiVersion":"v1","metadata":{},"items":[{"metadata":{"name":"old"}, + "spec":{"containers":[],"serviceAccountName":"sandbox"}}]}), + ); + assert_eq!( + migration::stop_legacy_consumer(&client, ®) + .await + .unwrap_err(), + migration::WAITING_FOR_CONSUMERS + ); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); + let authority = live::verify(&client, ®).await.unwrap(); + let sa = credentials::ensure_service_account(&client, ®, &authority) + .await + .unwrap(); + credentials::ensure_for_test(&client, ®, &authority, &sa) + .await + .unwrap(); + credentials::retire(&client, ®).await.unwrap(); + let locked = state.lock().unwrap(); + assert!( + !locked + .objects + .contains_key("/api/v1/namespaces/kars-sre/secrets/sre-api-router-identity") + ); + assert!( + locked + .calls + .iter() + .filter(|(method, _, _)| method == "DELETE") + .all( + |(_, _, body)| body["preconditions"]["uid"].as_str().is_some() + && body["preconditions"]["resourceVersion"].as_str().is_some() + ) + ); +} + +#[tokio::test] +async fn old_control_token_consumers_must_disappear_even_after_rollout_counters_converge() { + let (_server, client, state) = fixture().await; + let reg = registration(); + let epoch = reg.epoch(); + { + let mut locked = state.lock().unwrap(); + locked.objects.insert("/api/v1/secrets".into(),json!({ + "kind":"SecretList","apiVersion":"v1","metadata":{},"items":[{ + "kind":"Secret","apiVersion":"v1","metadata":{"name":"router-services-admin","namespace":"kars-sre","uid":"control","resourceVersion":"1", + "labels":{"app.kubernetes.io/managed-by":"kars-controller"}, + "annotations":{"kars.azure.com/sandbox-uid":"source","kars.azure.com/namespace-uid":"runtime-ns",EPOCH:epoch}}}]})); + locked.objects.insert("/apis/apps/v1/namespaces/kars-sre/deployments/sre".into(),json!({ + "metadata":{"name":"sre","namespace":"kars-sre","uid":"consumer","resourceVersion":"1","generation":2, + "labels":{"kars.azure.com/sandbox":"sre"}, + "managedFields":[{"manager":crate::field_managers::CLAWSANDBOX,"operation":"Apply","apiVersion":"apps/v1", + "fieldsType":"FieldsV1","fieldsV1":{"f:spec":{}}}]}, + "spec":{"replicas":0,"selector":{"matchLabels":{"app":"sre"}},"template":{"metadata":{"annotations":{EPOCH:epoch}}, + "spec":{"containers":[]}}}, + "status":{"observedGeneration":2,"updatedReplicas":0,"availableReplicas":0}})); + locked.objects.insert( + "/api/v1/namespaces/kars-sre/pods".into(), + json!({ + "kind":"PodList","apiVersion":"v1","metadata":{},"items":[{"metadata":{"name":"old-router", + "deletionTimestamp":"2026-09-08T00:00:00Z"},"spec":{"containers":[]}}]}), + ); + } + let error = migration::rotate_owned_control_credentials(&client, ®) + .await + .unwrap_err(); + assert!(migration::is_waiting(&error)); + state + .lock() + .unwrap() + .objects + .get_mut("/api/v1/namespaces/kars-sre/pods") + .unwrap()["items"] = json!([]); + migration::rotate_owned_control_credentials(&client, ®) + .await + .unwrap(); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, path, _)| method == "GET" || path.ends_with("/subjectaccessreviews")) + ); +} diff --git a/controller/src/sre_registration.rs b/controller/src/sre_registration.rs new file mode 100644 index 000000000..93e7861b3 --- /dev/null +++ b/controller/src/sre_registration.rs @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Cluster-scoped operator authorization for the canonical SRE identity. +//! Namespace ownership identifies an occupant; only this resource delegates +//! privileged SRE authority to that exact occupant. + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub const NAME: &str = "canonical"; +pub const ROUTER_SA: &str = "sre-api-router"; +pub const RUNTIME_NAMESPACE: &str = "kars-sre"; +pub const PRIVATE_SECRET: &str = "sre-api-router-identity"; +pub const AGENT_SECRET: &str = "sre-api-agent"; +pub const OWNER: &str = "kars.azure.com/sre-registration-uid"; +pub const EPOCH: &str = "kars.azure.com/sre-privacy-epoch"; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct NamedUid { + pub name: String, + pub uid: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Source { + pub namespace: String, + pub name: String, + pub uid: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ControllerIdentity { + pub namespace: NamedUid, + pub deployment: NamedUid, + pub release: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct BindingReview { + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, + pub name: String, + pub uid: String, + pub resource_version: String, + pub role_ref: k8s_openapi::api::rbac::v1::RoleRef, + pub subjects: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ConsumerReview { + pub namespace: String, + pub name: String, + pub uid: String, + pub resource_version: String, +} + +#[derive(CustomResource, Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[kube( + group = "kars.azure.com", + version = "v1alpha1", + kind = "KarsSRERegistration", + plural = "karssreregistrations", + status = "RegistrationStatus", + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"# +)] +#[serde(rename_all = "camelCase")] +pub struct KarsSRERegistrationSpec { + pub controller: ControllerIdentity, + pub sandbox: Source, + pub runtime_namespace: NamedUid, + #[serde(default)] + pub legacy_bindings: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub legacy_consumer: Option, + #[serde(default = "enabled")] + pub enabled: bool, +} + +fn enabled() -> bool { + true +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct RegistrationStatus { + pub phase: String, + pub observed_generation: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub privacy_epoch: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_service_account_uid: Option, + #[serde(default)] + pub legacy_secret_access_denied: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub privacy_revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +impl KarsSRERegistration { + pub fn validate(&self) -> Result<(), String> { + let label = |value: &str| { + !value.is_empty() + && value.len() <= 63 + && value + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + && value.as_bytes()[0].is_ascii_alphanumeric() + && value.as_bytes()[value.len() - 1].is_ascii_alphanumeric() + }; + if self.metadata.name.as_deref() != Some(NAME) + || self.spec.sandbox.name != "sre" + || self.spec.runtime_namespace.name != RUNTIME_NAMESPACE + || !label(&self.spec.sandbox.namespace) + || self.spec.sandbox.namespace != self.spec.controller.namespace.name + || self.spec.controller.deployment.name != "kars-controller" + || self.spec.controller.release.is_empty() + { + return Err( + "Registration must pin the canonical SRE and its controller/release namespace" + .into(), + ); + } + for uid in [ + self.metadata.uid.as_deref().unwrap_or_default(), + &self.spec.controller.namespace.uid, + &self.spec.controller.deployment.uid, + &self.spec.sandbox.uid, + &self.spec.runtime_namespace.uid, + ] { + if uid.trim().is_empty() { + return Err("Registration identity omitted an exact UID".into()); + } + } + if self.metadata.deletion_timestamp.is_some() { + return Err( + "Registration is terminating; disable and retire it before deletion".into(), + ); + } + let mut bindings = std::collections::BTreeSet::new(); + for review in &self.spec.legacy_bindings { + if !matches!(review.kind.as_str(), "ClusterRoleBinding" | "RoleBinding") + || review.name.is_empty() + || review.uid.is_empty() + || review.resource_version.is_empty() + || (review.kind == "RoleBinding") + != review.namespace.as_ref().is_some_and(|ns| label(ns)) + || !bindings.insert(( + review.kind.clone(), + review.namespace.clone(), + review.name.clone(), + )) + { + return Err( + "Legacy binding reviews require unique exact names, UIDs and resourceVersions" + .into(), + ); + } + } + if let Some(consumer) = &self.spec.legacy_consumer + && (consumer.namespace != RUNTIME_NAMESPACE + || consumer.name != "sre" + || consumer.uid.is_empty() + || consumer.resource_version.is_empty()) + { + return Err( + "Legacy SRE consumer review is incomplete or targets another workload".into(), + ); + } + Ok(()) + } + + pub fn epoch(&self) -> String { + let mut value = serde_json::json!({ + "domain":crate::sre_privacy::REVISION, + "uid":self.metadata.uid, "generation":self.metadata.generation, "spec":self.spec, + }); + value.sort_all_objects(); + crate::providers::signing::sha256_hex( + &serde_json::to_vec(&value).expect("registration serializes"), + ) + } +} diff --git a/deploy/helm/kars/templates/crd-karssreregistration.yaml b/deploy/helm/kars/templates/crd-karssreregistration.yaml new file mode 100644 index 000000000..cc9374408 --- /dev/null +++ b/deploy/helm/kars/templates/crd-karssreregistration.yaml @@ -0,0 +1,115 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: karssreregistrations.kars.azure.com + annotations: + helm.sh/resource-policy: keep +spec: + group: kars.azure.com + scope: Cluster + names: + kind: KarsSRERegistration + plural: karssreregistrations + singular: karssreregistration + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + x-kubernetes-validations: + - rule: "self.metadata.name == 'canonical'" + message: "The canonical SRE registration is the only supported instance" + properties: + apiVersion: {type: string} + kind: {type: string} + metadata: {type: object} + spec: + type: object + required: [controller, sandbox, runtimeNamespace] + properties: + enabled: {type: boolean, default: true} + controller: + type: object + required: [namespace, deployment, release] + properties: + namespace: + type: object + required: [name, uid] + properties: + name: {type: string, minLength: 1} + uid: {type: string, minLength: 1} + deployment: + type: object + required: [name, uid] + properties: + name: {type: string, enum: [kars-controller]} + uid: {type: string, minLength: 1} + release: {type: string, minLength: 1, maxLength: 53} + sandbox: + type: object + required: [namespace, name, uid] + properties: + namespace: {type: string, minLength: 1} + name: {type: string, enum: [sre]} + uid: {type: string, minLength: 1} + runtimeNamespace: + type: object + required: [name, uid] + properties: + name: {type: string, enum: [kars-sre]} + uid: {type: string, minLength: 1} + legacyConsumer: + type: object + required: [namespace, name, uid, resourceVersion] + properties: + namespace: {type: string, enum: [kars-sre]} + name: {type: string, enum: [sre]} + uid: {type: string, minLength: 1} + resourceVersion: {type: string, minLength: 1} + legacyBindings: + type: array + maxItems: 32 + items: + type: object + required: [kind, name, uid, resourceVersion, roleRef, subjects] + properties: + kind: {type: string, enum: [ClusterRoleBinding, RoleBinding]} + namespace: {type: string} + name: {type: string, minLength: 1} + uid: {type: string, minLength: 1} + resourceVersion: {type: string, minLength: 1} + roleRef: + type: object + required: [apiGroup, kind, name] + properties: + apiGroup: {type: string, enum: [rbac.authorization.k8s.io]} + kind: {type: string, enum: [Role, ClusterRole]} + name: {type: string, minLength: 1} + subjects: + type: array + items: + type: object + required: [kind, name] + properties: + apiGroup: {type: string} + kind: {type: string} + name: {type: string} + namespace: {type: string} + x-kubernetes-validations: + - rule: "self.sandbox.namespace == self.controller.namespace.name" + message: "SRE must be registered in its controller/release namespace" + status: + type: object + properties: + phase: {type: string} + observedGeneration: {type: integer, format: int64} + privacyEpoch: {type: string} + routerServiceAccountUid: {type: string} + legacySecretAccessDenied: {type: boolean} + privacyRevision: {type: string} + detail: {type: string} diff --git a/deploy/helm/kars/templates/sre-authority-admission.yaml b/deploy/helm/kars/templates/sre-authority-admission.yaml new file mode 100644 index 000000000..c7b3e1a62 --- /dev/null +++ b/deploy/helm/kars/templates/sre-authority-admission.yaml @@ -0,0 +1,329 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-source-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["karssandboxes", "karssandboxes/status"] + matchConditions: + - name: reserved-sre-source + expression: >- + (object != null && (object.metadata.name == 'sre' || + (has(object.metadata.labels) && 'kars.azure.com/role' in object.metadata.labels && + object.metadata.labels['kars.azure.com/role'] == 'sre'))) || + (oldObject != null && (oldObject.metadata.name == 'sre' || + (has(oldObject.metadata.labels) && 'kars.azure.com/role' in oldObject.metadata.labels && + oldObject.metadata.labels['kars.azure.com/role'] == 'sre'))) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "Canonical SRE source changes require explicit cluster-level registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-source-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-source-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-registration-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["karssreregistrations", "karssreregistrations/status"] + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "SRE enrollment requires explicit registrar use permission" + reason: Forbidden + - expression: >- + request.operation != 'DELETE' || + (oldObject.spec.enabled == false && oldObject.?status.?phase.orValue('') == 'Retired') + message: "Disable and retire SRE authority before deleting its registration" + reason: Forbidden + - expression: >- + object == null || oldObject == null || + !has(oldObject.status) || !has(oldObject.status.routerServiceAccountUid) || + oldObject.status.phase == 'Retired' || + (object.spec.controller == oldObject.spec.controller && + object.spec.sandbox == oldObject.spec.sandbox && + object.spec.runtimeNamespace == oldObject.spec.runtimeNamespace) + message: "Retire issued SRE authority before changing its bound identities" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-registration-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-registration-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-identity + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["serviceaccounts", "serviceaccounts/token"] + matchConditions: + - name: reserved-router-identity + expression: >- + request.name == 'sre-api-router' || + (object != null && object.metadata.name == 'sre-api-router') || + (oldObject != null && oldObject.metadata.name == 'sre-api-router') + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() || + (request.subResource == 'token' && + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('renew').allowed()) + message: "Reserved SRE router identity requires registrar use; its TokenRequest renewal requires explicit renew authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-identity + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-identity + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-binding-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["rbac.authorization.k8s.io"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["clusterrolebindings", "rolebindings"] + variables: + - name: oldSubjects + expression: "oldObject == null ? [] : oldObject.?subjects.orValue([])" + - name: newSubjects + expression: "object == null ? [] : object.?subjects.orValue([])" + matchConditions: + - name: sre-binding + expression: >- + (object != null && object.?subjects.orValue([]).exists(s, + (s.kind == 'ServiceAccount' && s.?namespace.orValue('') == 'kars-sre' && + s.name in ['sandbox', 'sre-api-router']) || + (s.kind == 'User' && s.name in + ['system:serviceaccount:kars-sre:sandbox', 'system:serviceaccount:kars-sre:sre-api-router']) || + (s.kind == 'Group' && s.name == 'system:serviceaccounts:kars-sre'))) || + (oldObject != null && oldObject.?subjects.orValue([]).exists(s, + (s.kind == 'ServiceAccount' && s.?namespace.orValue('') == 'kars-sre' && + s.name in ['sandbox', 'sre-api-router']) || + (s.kind == 'User' && s.name in + ['system:serviceaccount:kars-sre:sandbox', 'system:serviceaccount:kars-sre:sre-api-router']) || + (s.kind == 'Group' && s.name == 'system:serviceaccounts:kars-sre'))) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "SRE identity bindings require registrar authority" + reason: Forbidden + - expression: >- + object == null || + object.roleRef.name == 'kars-sandbox-spawner' || + !variables.newSubjects.exists(s, + (s.kind == 'ServiceAccount' && s.?namespace.orValue('') == 'kars-sre' && s.name == 'sandbox') || + (s.kind == 'User' && s.name == 'system:serviceaccount:kars-sre:sandbox') || + (s.kind == 'Group' && s.name == 'system:serviceaccounts:kars-sre')) || + (oldObject != null && object.roleRef == oldObject.roleRef && + variables.newSubjects == variables.oldSubjects) + message: "Legacy SRE agent grants may be retained unchanged for reviewed migration, never created or restored" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-binding-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-binding-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-material + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["secrets"] + matchConditions: + - name: protected-material + expression: >- + request.namespace == 'kars-sre' && + ((object != null && object.metadata.name in ['sre-api-router-identity', 'sre-api-agent']) || + (oldObject != null && oldObject.metadata.name in ['sre-api-router-identity', 'sre-api-agent'])) + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "SRE identity material requires cluster registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-material + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-material + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-no-legacy-tokens + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["secrets"] + matchConditions: + - name: private-runtime + expression: "request.namespace == 'kars-sre'" + validations: + - expression: >- + !(object != null && object.?type.orValue('') == 'kubernetes.io/service-account-token' && + has(object.metadata.annotations) && + 'kubernetes.io/service-account.name' in object.metadata.annotations && + object.metadata.annotations['kubernetes.io/service-account.name'] == 'sre-api-router') && + !(oldObject != null && oldObject.?type.orValue('') == 'kubernetes.io/service-account-token' && + has(oldObject.metadata.annotations) && + 'kubernetes.io/service-account.name' in oldObject.metadata.annotations && + oldObject.metadata.annotations['kubernetes.io/service-account.name'] == 'sre-api-router') + message: "The private SRE identity exclusively uses bound TokenRequests; legacy token Secrets are forbidden for every caller" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-no-legacy-tokens + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-no-legacy-tokens + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-source-retirement + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + paramKind: + apiVersion: kars.azure.com/v1alpha1 + kind: KarsSRERegistration + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["DELETE"] + resources: ["karssandboxes"] + validations: + - expression: >- + oldObject.metadata.uid != params.spec.sandbox.uid || + (params.spec.enabled == false && params.?status.?phase.orValue('') == 'Retired') + message: "Retire registered SRE authority before deleting its source" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-source-retirement + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-source-retirement + paramRef: + name: canonical + parameterNotFoundAction: Allow + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-pending-proposals + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["kars.azure.com"] + apiVersions: ["v1alpha1"] + operations: ["CREATE"] + resources: ["karssreactions"] + validations: + - expression: "object.spec.approval.state == 'Pending'" + message: "SRE actions must be created Pending; approval is a separate operator action" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-pending-proposals + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-pending-proposals + validationActions: [Deny, Audit] diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml new file mode 100644 index 000000000..a53da422f --- /dev/null +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -0,0 +1,221 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-consumer-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["deployments", "deployments/scale"] + matchConditions: + - name: canonical-runtime-consumer + expression: "request.namespace == 'kars-sre' && request.name == 'sre'" + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "Canonical SRE runtime changes require registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-consumer-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-consumer-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-role-authority + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["rbac.authorization.k8s.io"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE", "DELETE"] + resources: ["roles", "clusterroles"] + matchConditions: + - name: reserved-sre-role + expression: >- + request.name in ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', 'kars-sre-private-diagnostics', + 'kars-sre-registrar', 'kars-sre-retired-agent'] || + (request.namespace == 'kars-sre' && request.name == 'sre-api-self-renew') + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations') + .name('canonical').check('use').allowed() + message: "Reserved SRE authority roles require explicit registrar use permission" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-role-authority + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-role-authority + validationActions: [Deny, Audit] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-mounts + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["pods", "pods/ephemeralcontainers"] + matchConditions: + - name: sre-runtime + expression: "request.namespace == 'kars-sre'" + variables: + - name: containers + expression: >- + object.spec.containers + object.spec.?initContainers.orValue([]) + + object.spec.?ephemeralContainers.orValue([]) + - name: privateMaterial + expression: >- + object.spec.?volumes.orValue([]).exists(v, + (has(v.secret) && v.secret.secretName == 'sre-api-router-identity') || + (has(v.projected) && v.projected.sources.exists(s, + has(s.secret) && s.secret.name == 'sre-api-router-identity'))) || + variables.containers.exists(c, + c.?envFrom.orValue([]).exists(e, has(e.secretRef) && e.secretRef.name == 'sre-api-router-identity') || + c.?env.orValue([]).exists(e, has(e.valueFrom) && has(e.valueFrom.secretKeyRef) && + e.valueFrom.secretKeyRef.name == 'sre-api-router-identity')) || + object.spec.?serviceAccountName.orValue('') == 'sre-api-router' + validations: + - expression: >- + !variables.privateMaterial || + authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() || + authorizer.group('').resource('pods').check('create').allowed() + message: "Private SRE material may only be mounted by registrars or cluster-wide workload controllers" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-mounts + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-mounts + validationActions: [Deny, Audit] +--- +# Pod admission must also validate parent templates: otherwise a namespaced +# caller could launder a private mount through the privileged ReplicaSet/Job controller. +{{ range $kind := list "workloads" "cronjobs" }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-{{ $kind }} + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + {{- if eq $kind "workloads" }} + - apiGroups: ["apps"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] + - apiGroups: ["batch"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["jobs"] + {{- else }} + - apiGroups: ["batch"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["cronjobs"] + {{- end }} + matchConditions: + - name: sre-runtime + expression: "request.namespace == 'kars-sre'" + variables: + - name: pod + expression: {{ if eq $kind "cronjobs" }}"object.spec.jobTemplate.spec.template.spec"{{ else }}"object.spec.template.spec"{{ end }} + - name: containers + expression: "variables.pod.containers + variables.pod.?initContainers.orValue([])" + - name: privateMaterial + expression: >- + variables.pod.?volumes.orValue([]).exists(v, + (has(v.secret) && v.secret.secretName == 'sre-api-router-identity') || + (has(v.projected) && v.projected.sources.exists(s, + has(s.secret) && s.secret.name == 'sre-api-router-identity'))) || + variables.containers.exists(c, + c.?envFrom.orValue([]).exists(e, has(e.secretRef) && e.secretRef.name == 'sre-api-router-identity') || + c.?env.orValue([]).exists(e, has(e.valueFrom) && has(e.valueFrom.secretKeyRef) && + e.valueFrom.secretKeyRef.name == 'sre-api-router-identity')) || + variables.pod.?serviceAccountName.orValue('') == 'sre-api-router' + validations: + - expression: >- + !variables.privateMaterial || + authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() || + authorizer.group('').resource('pods').check('create').allowed() + message: "Private SRE workload templates require registrar or cluster-wide workload-controller authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-{{ $kind }} + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-{{ $kind }} + validationActions: [Deny, Audit] +{{ end }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: kars-sre-private-connect + annotations: + helm.sh/resource-policy: keep +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CONNECT"] + resources: ["pods/exec", "pods/attach", "pods/portforward", "pods/proxy"] + matchConditions: + - name: sre-runtime + expression: "request.namespace == 'kars-sre'" + validations: + - expression: >- + authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() + message: "Connecting into the SRE private-identity runtime requires registrar authority" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: kars-sre-private-connect + annotations: + helm.sh/resource-policy: keep +spec: + policyName: kars-sre-private-connect + validationActions: [Deny, Audit] diff --git a/deploy/helm/kars/templates/sre-authority-rbac.yaml b/deploy/helm/kars/templates/sre-authority-rbac.yaml new file mode 100644 index 000000000..27ad29555 --- /dev/null +++ b/deploy/helm/kars/templates/sre-authority-rbac.yaml @@ -0,0 +1,138 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-registrar + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + resourceNames: ["canonical"] + verbs: ["use", "update", "patch", "delete"] +--- +# Deliberately unbound. Only a cluster administrator delegates registrar power. +# Controller read/status/use permissions do not permit it to enroll a tenant. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-router-renew + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + resourceNames: ["canonical"] + verbs: ["get", "renew"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-retired-agent + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: [] +--- +# Unbound in the chart. The registration controller grants this only to the +# exact claimed runtime namespace's protected, UID-owned router identity. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-private-diagnostics + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssandboxes", "inferencepolicies", "toolpolicies", "egressapprovals", + "karsmemories", "karsevals", "trustgraphs", "karspairings", "a2aagents", "mcpservers", + "karsauthconfigs", "karstasks", "karsteams", "karsprofiles", "karsskills", "karsreceipts", "karsapprovals"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods", "pods/log", "services", "configmaps", "events", "namespaces", + "serviceaccounts", "nodes", "endpoints", "resourcequotas", "limitranges", "secrets"] + verbs: ["get", "list"] + - apiGroups: ["apps"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"] + verbs: ["get", "list"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies", "ingresses"] + verbs: ["get", "list"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list"] + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["get", "list"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kars-sre-authority-controller + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +rules: + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + verbs: ["get", "list", "watch"] + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations"] + resourceNames: ["canonical"] + verbs: ["use"] + - apiGroups: ["kars.azure.com"] + resources: ["karssreregistrations/status"] + resourceNames: ["canonical"] + verbs: ["get", "patch", "update"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "roles", "rolebindings"] + verbs: ["get", "list", "watch"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["create", "patch", "update", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + resourceNames: ["kars-sre-private-diagnostics", "kars-sre-action-author", "kars-sre-router-renew"] + verbs: ["bind"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kars-sre-authority-controller + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: sre-authority +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kars-sre-authority-controller +subjects: + - kind: ServiceAccount + name: kars-controller + namespace: {{ .Release.Namespace }} diff --git a/deploy/helm/kars/templates/sre.yaml b/deploy/helm/kars/templates/sre.yaml index 6f87944da..5fef64b6d 100644 --- a/deploy/helm/kars/templates/sre.yaml +++ b/deploy/helm/kars/templates/sre.yaml @@ -1,10 +1,9 @@ {{- /* kars-sre — the built-in SRE agent (Slice 1 MVP). -Gated on `.Values.sre.enabled` (default: false). Enable via: - helm upgrade --reuse-values --set sre.enabled=true ... -or — preferred — via the CLI: - kars sre install +Gated on `.Values.sre.enabled` (default: false). `kars sre install` creates +and enrolls a fresh source. Existing installs require reviewed authority +migration; Helm/GitOps users follow docs/how-to/sre-authority.md. What this template creates (when sre.enabled=true): - InferencePolicy `sre-inference` (Release.Namespace) @@ -14,15 +13,14 @@ What this template creates (when sre.enabled=true): `sre.py` but only registers its tools when this env is set — standard Hermes sandboxes don't get the SRE tool surface) - ClusterRole `kars-sre-reader` — kars-CR read scope (Slice 1) - - ClusterRoleBinding `kars-sre-reader` — bound to the SA - `sandbox` in namespace `kars-sre` (the controller-created default) + - No fresh privileged runtime binding. Registration reconciliation grants + only the separate router identity after verified legacy credential denial. - ToolPolicy `sre-tools` (Release.Namespace) — gates the sre_* tool surface Per design (docs/blueprints/07-kars-sre-proposal.md §7.8 — privilege containment): - - Sandbox uniqueness VAP (kars-sre-uniqueness) — Slice 1 ships the - label `kars.azure.com/role=sre`; the VAP itself lands in Slice 3 - alongside the typed apply-fix path + - Admission protects the registered source and private runtime identities; + labels alone never authorize SRE privileges. - kars_spawn family deregistered when KARS_SRE_ENABLED=true (enforced in the plugin __init__.py — §7.8.5) - kars_mesh_* family deregistered when KARS_SRE_ENABLED=true @@ -35,7 +33,28 @@ containment): {{- $retainNamespace := false }} {{- $legacyWriter := dict }} {{- $retainWriter := false }} +{{- $retainedBindings := list }} {{- if .Release.IsUpgrade }} +{{- range $name := list "kars-sre-reader" "kars-sre-action-author" }} +{{- $binding := lookup "rbac.authorization.k8s.io/v1" "ClusterRoleBinding" "" $name }} +{{- if $binding }} +{{- $annotations := default dict $binding.metadata.annotations }} +{{- if and (eq (index $annotations "meta.helm.sh/release-name") $.Release.Name) (eq (index $annotations "meta.helm.sh/release-namespace") $.Release.Namespace) }} +{{- range $subject := default list $binding.subjects }} +{{- if or (and (eq $subject.kind "ServiceAccount") (eq $subject.name "sandbox") (eq (default "" $subject.namespace) "kars-sre")) (and (eq $subject.kind "User") (eq $subject.name "system:serviceaccount:kars-sre:sandbox")) }} +{{- if not (($.Values.sre | default dict).authorityStage | default false) }} +{{- fail "Legacy SRE grants must be explicitly reviewed/enrolled and retired before this upgrade; run kars sre authority stage/preview/enroll/migrate" }} +{{- end }} +{{- end }} +{{- end }} +{{- $_ := set $binding.metadata "annotations" (mergeOverwrite (deepCopy $annotations) (dict "helm.sh/resource-policy" "keep")) }} +{{- $_ := unset $binding.metadata "managedFields" }} +{{- $retainedBindings = append $retainedBindings $binding }} +{{- else if ($.Values.sre | default dict).enabled }} +{{- fail "An existing SRE binding belongs to another release; no adoption or privilege grant is permitted" }} +{{- end }} +{{- end }} +{{- end }} {{- $legacyNamespace = lookup "v1" "Namespace" "" "kars-sre" }} {{- if $legacyNamespace }} {{- $annotations := default dict $legacyNamespace.metadata.annotations }} @@ -53,6 +72,12 @@ containment): {{- end }} {{- end }} {{- end }} +{{- range $binding := $retainedBindings }} +--- +# Keep the reviewed retired binding, preserving unrelated subjects and CAS +# identity. No fresh SRE privilege binding is emitted by Helm. +{{ toYaml $binding }} +{{- end }} {{- if $retainNamespace }} --- # Retain an earlier release's Namespace even when disabling SRE. Dropping it @@ -112,15 +137,27 @@ spec: dailyTokens: {{ (.Values.sre | default dict).dailyTokens | default 2000000 }} --- # kars-sre KarsSandbox — Hermes runtime, SRE plugin gated on env. +{{- if ((.Values.sre | default dict).authorityStage | default false) }} +{{- $source := lookup "kars.azure.com/v1alpha1" "KarsSandbox" .Release.Namespace "sre" }} +{{- if not $source }} +{{- fail "Authority staging cannot CREATE an enabled SRE source by name; stage disabled core, then use atomic authority stage-source" }} +{{- end }} +{{- $sourceAnnotations := default dict $source.metadata.annotations }} +{{- if not (and (eq (index $sourceAnnotations "meta.helm.sh/release-name") .Release.Name) (eq (index $sourceAnnotations "meta.helm.sh/release-namespace") .Release.Namespace)) }} +{{- fail "Authority staging cannot adopt an unowned SRE source; explicitly review its identity outside Helm" }} +{{- end }} +# Stage the controller/API without rewriting the source being reviewed. +{{- $retainedSource := omit (deepCopy $source) "status" }} +{{- $_ := unset $retainedSource.metadata "managedFields" }} +{{ toYaml $retainedSource }} +{{- else }} apiVersion: kars.azure.com/v1alpha1 kind: KarsSandbox metadata: name: sre namespace: {{ .Release.Namespace }} labels: - # The label the future kars-sre-uniqueness VAP keys on (Slice 3). - # Slice 1 ships the label so by-the-time-VAP-lands no operator can - # have applied a second role=sre sandbox first. + # Admission reserves SRE sources; this label alone never grants authority. kars.azure.com/role: sre kars.azure.com/channels: none app.kubernetes.io/name: kars @@ -194,7 +231,7 @@ spec: # registry. The SRE agent does not use the mesh (§7.8.6 — three # layers: spec, image plugin, networkPolicy; this is layer 3). allowedEndpoints: - # In-cluster apiserver — the SRE agent's primary counterparty. + # Router-only upstream; the agent uses the filtered loopback HTTPS API. - host: kubernetes.default.svc.cluster.local port: 443 # Telegram Bot API — required when the operator configures @@ -215,6 +252,7 @@ spec: port: {{ .port }} {{- end }} {{- end }} +{{- end }} --- # kars-sre ToolPolicy — gates the sre_* tool surface. # @@ -384,23 +422,6 @@ rules: # kubectl accepts CRBs that reference not-yet-existing SAs — the # binding activates when the SA appears on first sandbox # reconciliation. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kars-sre-reader - labels: - app.kubernetes.io/name: kars - app.kubernetes.io/component: sre - app.kubernetes.io/managed-by: {{ .Release.Service }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kars-sre-reader -subjects: - - kind: ServiceAccount - name: sandbox - namespace: kars-sre ---- # --------------------------------------------------------------------- # Slice 3 — Typed apply-fix path (KarsSREAction CRD + writer SA) # --------------------------------------------------------------------- @@ -487,23 +508,6 @@ rules: resources: ["karssreactions/status"] verbs: ["get"] --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kars-sre-action-author - labels: - app.kubernetes.io/name: kars - app.kubernetes.io/component: sre - app.kubernetes.io/managed-by: {{ .Release.Service }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kars-sre-action-author -subjects: - - kind: ServiceAccount - name: sandbox - namespace: kars-sre ---- # Operator-facing role. Cluster admin binds humans / groups to # this manually (e.g. # kubectl create clusterrolebinding sre-approvers \ diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md new file mode 100644 index 000000000..ed54bd189 --- /dev/null +++ b/docs/how-to/sre-authority.md @@ -0,0 +1,212 @@ +# Registered SRE authority and credential privacy + +SRE namespace occupancy is not authorization. The cluster-scoped +`KarsSRERegistration/canonical` is the operator trust root. It pins the exact +controller Deployment and controller namespace UIDs, owning release, canonical +`sre` Sandbox namespace/name/UID, and `kars-sre` namespace UID. + +The `kars-sre-registrar` ClusterRole is **unbound**. Only an explicit cluster +administrator delegation gives an operator registration permissions. +Namespace administrators, agents, and the Bridge BFF receive none by default. +Installers also need their normal core/Helm deployment permissions. +Kubernetes must support stable `admissionregistration.k8s.io/v1` +ValidatingAdmissionPolicy and CEL authorizer checks. The controller refuses +privilege until all fourteen policies are observed, type-checked without +warnings, and bound with `Deny`. + +## Fresh installation + +Install the core prerequisite controller and router first. SRE remains disabled +by default. A registrar can then run: + +```sh +kars sre install --namespace kars-system --release kars +``` + +The CLI stages only supporting policies and a genuinely new source CR, using +atomic CREATE and its returned UID. A racing existing CR is never fetched and +adopted. It waits for that UID's complete namespace claim and backlink before +enrollment. No privileged RoleBinding is emitted by Helm. + +If either the source or runtime namespace already exists, use explicit review +below instead. A foreign `kars-sre` occupant remains untouched and receives no +privilege. + +## Existing installation: stage, review, enroll + +Normal `up`, `up --upgrade`, `upgrade`, local-Kubernetes `dev`, and `push --apply` +stop before Kubernetes deployment changes when legacy SRE grants remain. +Staging is the explicit registrar operation that installs the prerequisite +controller/API while retaining old grants unchanged: + +```sh +kars sre authority stage --namespace kars-system --release kars \ + --controller-image : \ + --router-image : --dry-run +``` + +Review the output, then run the same command without `--dry-run`. Staging does +not enroll an occupant or issue private SRE credentials. + +```sh +kars sre authority preview --namespace kars-system --release kars +kars sre authority enroll --namespace kars-system --release kars \ + --sandbox-uid --namespace-uid \ + --binding 'ClusterRoleBinding//kars-sre-reader=@' \ + --binding 'ClusterRoleBinding//kars-sre-action-author=@' \ + --consumer '@' --dry-run +``` + +Use **every** binding review printed by preview, not just the illustrative +names above. Updating an existing registration additionally requires +`--registration-uid` and `--resource-version`. After review, omit `--dry-run` +and wait for migration: + +```sh +kars sre authority migrate --namespace kars-system --release kars +``` + +The controller validates the full review set before mutation, removes only the +legacy SRE subject from the reviewed bindings using UID/resourceVersion +preconditions, preserves unrelated subjects, and stops the reviewed old +consumer. Broad group grants or unreviewed/custom resources stop migration; +the operator must restructure those grants explicitly. + +Real Kubernetes authorization reviews must deny the old SRE principal Secret +get/list/watch access before private credentials are issued. The shared +`shared/sre_privacy.rs` contract checks both namespace and cluster scope, +including generic access and name-restricted access to the two protected +Secrets. Both issuance and each proxy authorization run these live checks; +an earlier `Ready` status boolean is not sufficient. Namespace/source +recreation, API errors, or incomplete claims fail closed. No ServiceAccount +delete/recreate shortcut revokes the legacy identity. + +The controller also rotates proven owned `router-services-admin` credentials, +if any exist, and restarts their owned consumers. Their absence on the +prerequisite base is normal. Later governed-service issuers must call +`sre_authority::privacy_epoch` immediately before issuance. + +### Legacy token Secrets and watch-only grants + +The private `sre-api-router` identity must **never** use legacy +`kubernetes.io/service-account-token` Secrets. Admission denies CREATE and +UPDATE for arbitrary Secret names when either the old or new object combines +that type with `kubernetes.io/service-account.name=sre-api-router`. There is no +registrar or Kubernetes token-controller exception. Normal bound TokenRequest +renewal remains supported. + +Before creating the private ServiceAccount, granting authority, or issuing +credentials, the controller inventories Secret **metadata only**. Prestaged +aliases are rejected even if already populated, carrying an obsolete SA UID, +or annotated as another type. A renamed alias retaining the current or +registered SA UID is also rejected. Incomplete inventories and API errors fail +closed. Unknown Secret values are never read for this inspection, logged, +adopted, or deleted; an operator must review and resolve the conflicting objects. + +Discovery after a prior `Ready` revokes owned grants, retires owned private +credentials/identity, and clears the published privacy evidence. Foreign +objects and unrelated binding subjects remain intact. The Pod's `sandbox` +ServiceAccount and Azure federated subject are never deleted or replaced. +Recovery requires clean live checks and new bound credentials; replacement of +the owned Secret UID invalidates tokens from a previous privacy epoch. That UID +also drives consumer rollout, independently of TLS expiry. + +Privacy revision `kars.azure.com/sre-privacy/v2` prevents an old get/list-only +status from authorizing the new proxy or issuance helper. Watch-only and +wildcard Secret grants are included in controller and CLI legacy inventory; +unreviewed/group grants block migration before private issuance. The router +identity alone receives narrowly scoped authorization-review creation +permission for the live check. This API is not exposed through the agent proxy. + +## Helm and GitOps sequencing + +1. Install the prerequisite core/chart with `sre.enabled=false` for a fresh + installation. For an existing Helm-owned enabled SRE, use the explicit + `authority stage` workflow above: server-side lookup retains the exact + legacy source and grant UIDs/resourceVersions rather than re-rendering or + replacing them. Do not use offline `helm template` to guess those identities. +2. Wait for the CRD and admission policies to converge. Delegate the unbound + registrar role explicitly to the operator performing enrollment. Do not + grant it to the tenant GitOps controller or SRE agent. +3. For a fresh source, run `kars sre authority stage-source`, which atomically + CREATEs the CR and reports its actual source/namespace UIDs. For an existing + source, run `authority preview` instead; no source adoption by name occurs. +4. Review the `authority enroll --dry-run` JSON and exact binding/consumer + reviews. CREATE that cluster-scoped registration under registrar authority, + or apply an explicitly reviewed UID/resourceVersion-fenced update. The CLI + performs these operations; a GitOps operator may commit the same reviewed + specification but must not omit the API identity checks when applying it. +5. Wait for `Ready` with matching `observedGeneration`, then enable + `sre.enabled=true,sre.authorityStage=false` in the managed release. + GitOps pruning must preserve the chart's retained authority, registration, + legacy-binding, and namespace resources through this sequence. + +Non-Helm/template installs use the same CLI stage/enroll workflow; staging +validates all authority resources first and CAS-updates only controller image +and router image configuration, retaining unrelated settings. Different +unowned authority objects require explicit operator resolution, not force +adoption. Standard install/upgrade commands are not a migration bypass. + +## Existing Hermes images remain compatible + +The Pod still uses ServiceAccount `sandbox`, preserving its Azure Workload +Identity federated subject and existing IMDS behavior. The router alone receives +the ordinary projected Kubernetes credential. A separate `sre-api-router` +identity, scoped to the registered namespace incarnation, performs diagnostic +API access and renews its own Secret-bound short-lived TokenRequests. + +The agent receives **no Kubernetes JWT**. Its existing paths contain an opaque, +agent-safe proxy credential, loopback CA, and namespace: + +- `/var/run/secrets/kubernetes.io/serviceaccount/token` +- `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` +- `/var/run/secrets/kubernetes.io/serviceaccount/namespace` + +Standard `KUBERNETES_SERVICE_HOST/PORT` point to `https://127.0.0.1:9446`. +Pinned Hermes clients continue using HTTPS, CA verification, raw pod-log GETs, +and proposal POSTs without an image-specific fallback. Azure token projection +is excluded from the agent container. The old apiserver egress bypass is gone. +Admission protects both direct Pod mounts and Deployment/ReplicaSet/Job and +CronJob templates from laundering a private mount through Kubernetes workload +controllers. Exec/attach/port-forward into the private SRE runtime requires +registrar authority. Cluster-wide workload controllers remain trusted; +installing a custom privileged controller is a cluster-operator action. + +The proxy checks current registration and live UID/claim authority. It permits +the bounded first-party diagnostic read/log/metrics paths and Pending-only +`KarsSREAction` creation in `kars-sre`. Secret responses retain key names but +exclude values, `stringData`, annotations, labels, and managed-field copies. +Encoded/noncanonical paths, watches, streaming log follow, token requests, +exec/proxy subresources, arbitrary writes, and non-JSON media escapes are denied. +The existing Hermes proposal builder's two diagnostic labels are accepted; +owner references, status injection, arbitrary metadata, and approval fields +other than exactly `{"state":"Pending"}` are rejected. + +Limits: 16 concurrent operations, 64 KiB proposal bodies, 8 MiB JSON responses, +and 256 KiB logs. First-party watchers poll rather than stream. Private +credential renewal never falls back to the agent or Pod's less-privileged +credential. TLS rotation changes the opaque token and CA together and rolls +the SRE consumer; the old client rebuilds its TLS client on token change. + +## Retirement and rollback + +```sh +kars sre authority retire --registration-uid --resource-version +kars sre uninstall --namespace kars-system --release kars +``` + +Retirement revokes owned private grants and credentials before source cleanup. +Uninstall/destroy refuse an active enrollment or unretired legacy grants. +Retired registrations remain audit records; recreating the source requires +explicit enrollment of the new UIDs (`authority stage-source` can atomically +stage a genuinely new source). + +Do not roll back to a release that restores agent-held Kubernetes credentials +or legacy SRE grants. CLI rollback is rejected while registration exists, and +retained admission policies reject restoration of the old privileged subject. +Use a reviewed roll-forward release. Do not prune retained authority/admission +objects through GitOps without an operator-reviewed retirement. + +This boundary assumes the controller namespace and runtime infrastructure are +operator controlled. Cluster administrators can change RBAC/admission itself; +deliberately bypassing those controls is not defended by a container mount. diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md new file mode 100644 index 000000000..94d649a8f --- /dev/null +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -0,0 +1,145 @@ +# Security audit — registered SRE credential authority + +Status: implemented and locally qualified candidate; pending real Kubernetes +admission/migration proof and independent review. **Not a sign-off.** + +## Scope and trust root + +This prerequisite introduces cluster-scoped `KarsSRERegistration/canonical`. +Only explicitly delegated registrars can author it; the controller can read, +use, and reconcile status. Namespace occupancy, SRE labels, account names, and +Helm-looking metadata are not privilege delegation. + +## Boundaries implemented + +- Exact source, controller/release, and runtime namespace UIDs are checked live. +- Reviewed legacy grants retire with UID/resourceVersion preconditions. + Unrelated subjects/resources are preserved; custom/group ambiguity blocks + before migration mutations. +- Shared live authorization reviews require Secret get/list/watch denial in + namespace and cluster scope, including protected name-restricted grants, + before issuance and proxy forwarding. Old status booleans are insufficient. +- The Pod's Azure identity remains unchanged. A separate private Kubernetes + identity renews its short-lived Secret-bound token without ambient fallback. +- Old Hermes clients use a real loopback TLS endpoint through their standard + token/CA/namespace paths, with an opaque non-Kubernetes credential. +- Secret projection omits values and annotation copies, retaining key names. + Reads/logs/metrics and Pending-only proposals use a strict route/query/media + allowlist; exec/proxy/token/write escapes are denied. +- Admission gates protect reserved sources, identities, grants, and material. + Retained policies prevent insecure legacy grant restoration on rollback. +- Owned potentially exposed control credentials rotate and their consumers + restart. No governed-service feature is disabled merely because SRE exists. +- Arbitrary-name legacy SA-token Secrets for the private identity are denied + on CREATE and old/new UPDATE transitions, without a token-controller or + registrar exception. Metadata-only prestage scans catch populated aliases, + annotation changes and old/current SA UIDs before identity/grant issuance. + Unsafe discovery after Ready revokes owned authority and retires owned + credentials; unknown Secrets, replacement SA UIDs and unrelated subjects + are preserved. Recovery rotates the bound Secret UID and consumer. +- Privacy revision `kars.azure.com/sre-privacy/v2` and the shared Rust wire + helper are used by controller issuance, the privacy-epoch accessor, and the + router's current-authority check. Bound TokenRequest renewal, pinned Hermes + HTTPS clients, and the Pod's Azure ServiceAccount identity are unchanged. + +## Qualification + +Tests cover registrar/UID checks, reviewed grant CAS and unrelated subjects, +legacy denial, projection/WI/pinned-image invariants, real TLS API filtering, +private token renewal, direct agent-credential rejection, and the unchanged +Python Hermes client. CLI tests cover explicit enrollment and racing CREATE. + +Measured local results: + +- 37 controller SRE tests passed, including live UID/claim rejection, + real SubjectAccessReview request handling, legacy binding CAS, private + Secret-bound TokenRequests, TLS rotation/idempotency, owned retirement, + admission-status rejection, terminating old control-token consumers despite + converged rollout counters, metadata-only legacy-token alias scans, + watch-only/wildcard grant rejection, post-Ready owned revocation, replacement + UID preservation, token-anchor recovery, and existing writer/egress regressions. +- Nine router tests passed, including real loopback TLS, direct fake-token + rejection by the test Kubernetes API, Secret/list redaction, logs/metrics/ + Pending proposals, token renewal, the unchanged `sre_kube.py` HTTPS client, + and the actual unchanged `sre.py` proposal builder including its labels. + The new negative matrix covers old Ready evidence, watch-only authorization, + arbitrary-name/current-UID aliases, and metadata API errors before forwarding. +- The earlier baseline passed 158 CLI/Helm tests. The two HIGH closures reran + 116 affected CLI/Helm tests, all passing, plus TypeScript typecheck. CLI lint + reports zero errors and 29 existing warnings outside the new helpers. +- CLI dependencies came from an existing verified local cache (Vitest 4.1.10); + no npm install/ci. Standard TLS additions resolved `rcgen 0.13.2` and + `yasna 0.5.2`; no custom cryptography. + +- Final strict controller/router all-targets Clippy passed with `-D warnings`. + All 46 Rust SRE tests passed after both HIGH closures, including the original + readiness-capacity, proposal-builder compatibility, and spawner invariants. + Scoped rustfmt, whitespace checks, current file caps, and source headers pass. + +Parent dependency hygiene restored unrelated pre-existing `windows-sys` and +`oauth2`/`base64` resolution edges. The lockfile now adds only the required +direct TLS references and the new `rcgen`/`yasna` packages. Cargo accepted that +graph with `--offline --locked`; all 35 SRE tests and combined strict Clippy +passed again. An active disk guard protected that rerun; 18.35 GiB remained. + +Reproducible final Clippy command (both packages together, default features; +no `--features` or `--no-default-features` flags): + +```sh +CARGO_TARGET_DIR="${CARGO_TARGET_DIR:?Set the existing shared Cargo target first}" \ +CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS=2 \ +cargo clippy --offline --locked \ + -p kars-controller -p kars-inference-router --all-targets -- -D warnings +``` + +The final bounded Clippy run passed in 24.58 seconds with 17.91 GiB free before and +after. It used the same combined package/feature selection as the completed +Rust test command, without rerunning tests or changing feature variants: + +```sh +CARGO_TARGET_DIR="${CARGO_TARGET_DIR:?Set the existing shared Cargo target first}" \ +CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS=2 \ +cargo test --offline --locked \ + -p kars-controller -p kars-inference-router sre_ --quiet +``` + +Resource note: an earlier combined-package Cargo feature-unification run +unexpectedly expanded the shared target from 8.3 to 6.0 GiB free. Cargo was +stopped after that completed command; no shared artifacts were deleted by +this implementation task. After disk capacity recovered to 19.92 GiB, final +qualification used the existing exclusive root target and an active 8.5 GiB +process-stop threshold. The HIGH follow-up finished with 17.91 GiB free and +no Cargo processes. Parent's lockfile minimization was preserved byte-for-byte +(SHA-256 `8be5db8d96793bf2aaa73f14ab51463fb922a07011b5cfb53c341ae46a1f1ccc`); +no dependency resolution or feature-selection changes were performed. +The final CLI/chart state also preserves live SRE source UID/resourceVersion +during explicit Helm staging and retains the registration CRD on uninstall. +The existing Helm legacy-floor suite passed another ten tests, enabled/default +Helm lint passed, and no new Rust/TypeScript module exceeds 800 lines. + +The tests use real local TLS plus an HTTP test Kubernetes API, not a real +Kubernetes authorizer/admission server. Parent-owned API/E2E proof must verify +CEL type checking, the full staged/legacy migration and grant issuance, +tenant denial (including workload-template and connect escapes), token/TLS +rotation, and retirement on an actual supported Kubernetes release. +Later Azure/kars#550 must call the privacy-epoch helper before operator-secret issuance +and incorporate its epoch into cached-token rotation. No public image or +release is claimed qualified by this audit; no human signatures are supplied. + +The completed Kind acceptance harness now stages immutable historical fixtures +before the new admission policies, then exercises delegated registration, +legacy token-controller aliases, watch-only grants, reviewed migration ordering, +filtered TLS access through the unchanged Hermes client, and retirement/fresh +re-enrollment. Ten pure harness tests and shell syntax pass; actual cluster +execution remains pending. + +Harness construction exposed a CLI mismatch: a merge update could retain an +omitted retired `legacyConsumer`. Re-enrollment now tests the registration UID +and resourceVersion and replaces the complete reviewed spec with JSON Patch. +Absent and present consumer reviews plus stale registration identity are covered +by targeted regressions. The persisted-spec assertion in the real harness +remains strict. + +The shared Rust privacy helper is included in capability-audit, no-stub, +no-custom-crypto and runtime-affecting Kind path classification. Its location +outside the individual crates is not a security-gate exception. diff --git a/inference-router/Cargo.toml b/inference-router/Cargo.toml index d0bf4fb65..726c9fb87 100644 --- a/inference-router/Cargo.toml +++ b/inference-router/Cargo.toml @@ -12,6 +12,8 @@ rust-version.workspace = true axum.workspace = true reqwest.workspace = true rustls.workspace = true +tokio-rustls.workspace = true +rustls-pemfile.workspace = true tower.workspace = true bytes = "1" @@ -76,6 +78,7 @@ flate2 = "1" ed25519-dalek.workspace = true [dev-dependencies] +rcgen.workspace = true # Property-based testing (s5) — inline #[cfg(test)] proptest! blocks exercise # parsers/sanitizers with generated inputs. Shrinks counterexamples automatically. proptest = "1" diff --git a/inference-router/src/lib.rs b/inference-router/src/lib.rs index a8cc38a83..4b5e95ee8 100644 --- a/inference-router/src/lib.rs +++ b/inference-router/src/lib.rs @@ -49,6 +49,9 @@ pub mod routes; pub mod safety; pub mod sidecar_client; pub mod spawn; +#[path = "../../shared/sre_privacy.rs"] +mod sre_privacy; +pub mod sre_proxy; pub mod telemetry; /// Select RustCrypto for JWT signing and verification. Workspace builds also diff --git a/inference-router/src/main.rs b/inference-router/src/main.rs index 27d6809dc..a03b90377 100644 --- a/inference-router/src/main.rs +++ b/inference-router/src/main.rs @@ -80,6 +80,11 @@ async fn main() -> Result<()> { // that broke when the runtime image went distroless (#383). { let args: Vec = std::env::args().collect(); + if args.get(1).map(String::as_str) == Some("sre-ready") { + std::process::exit(i32::from( + !kars_inference_router::sre_proxy::readiness_probe().await, + )); + } if args.get(1).map(String::as_str) == Some("probe") { // Forms: `probe ` (GET) | `probe GET|POST [json-body]`. let (method, raw_path, body) = match (args.get(2), args.get(3)) { @@ -192,6 +197,9 @@ async fn main() -> Result<()> { } let state = routes::AppState::new(&config).await?; + let _sre_proxy = kars_inference_router::sre_proxy::start() + .await + .map_err(anyhow::Error::msg)?; // Start policy hot-reload watcher (polls AGT_POLICY_DIR for mtime changes). governance::Governance::spawn_policy_watcher(state.governance.clone()); diff --git a/inference-router/src/sre_proxy/backend.rs b/inference-router/src/sre_proxy/backend.rs new file mode 100644 index 000000000..0cce84cf2 --- /dev/null +++ b/inference-router/src/sre_proxy/backend.rs @@ -0,0 +1,365 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::sync::Mutex; + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Source { + pub namespace: String, + pub name: String, + pub uid: String, +} + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Config { + pub schema: String, + pub kube_url: String, + pub registration_uid: String, + pub privacy_epoch: String, + pub source: Source, + pub runtime_namespace: String, + pub namespace_uid: String, + pub service_account_uid: String, + pub secret_uid: String, +} + +struct Token { + value: String, + expiry: DateTime, +} + +pub(super) struct Backend { + pub config: Config, + client: reqwest::Client, + token: Mutex, + directory: PathBuf, +} + +fn token_files(directory: &Path) -> Result { + let value = std::fs::read_to_string(directory.join("kube-token")) + .map_err(|_| "Private SRE token file unavailable")?; + let expiry = std::fs::read_to_string(directory.join("kube-expires-at")) + .map_err(|_| "Private SRE expiry file unavailable")?; + let expiry = DateTime::parse_from_rfc3339(expiry.trim()) + .map_err(|_| "Private SRE expiry is invalid")? + .with_timezone(&Utc); + if value.trim().is_empty() { + return Err("Private SRE token is empty".into()); + } + Ok(Token { + value: value.trim().into(), + expiry, + }) +} + +impl Backend { + #[cfg(test)] + pub(super) fn for_test(config: Config, directory: PathBuf, expiry: DateTime) -> Arc { + Arc::new(Self { + config, + client: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(), + token: Mutex::new(Token { + value: "private-kubernetes-token".into(), + expiry, + }), + directory, + }) + } + + pub(super) fn load(directory: &Path) -> Result, String> { + let config: Config = serde_json::from_slice( + &std::fs::read(directory.join("config.json")) + .map_err(|_| "Private SRE configuration unavailable")?, + ) + .map_err(|_| "Private SRE configuration invalid")?; + let url = + reqwest::Url::parse(&config.kube_url).map_err(|_| "Private Kubernetes URL invalid")?; + if config.schema != "kars.azure.com/sre-api/v1" + || config.runtime_namespace != "kars-sre" + || config.source.name != "sre" + || url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || [ + config.registration_uid.as_str(), + config.privacy_epoch.as_str(), + config.source.namespace.as_str(), + config.source.uid.as_str(), + config.namespace_uid.as_str(), + config.service_account_uid.as_str(), + config.secret_uid.as_str(), + ] + .iter() + .any(|value| value.is_empty()) + { + return Err("Private SRE identity is incomplete or invalid".into()); + } + let ca = std::fs::read(directory.join("kube-ca.crt")) + .map_err(|_| "Kubernetes CA unavailable")?; + let certificate = + reqwest::Certificate::from_pem(&ca).map_err(|_| "Kubernetes CA invalid")?; + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(certificate) + .https_only(true) + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(20)) + .build() + .map_err(|_| "SRE Kubernetes client could not initialize")?; + Ok(Arc::new(Self { + config, + client, + token: Mutex::new(token_files(directory)?), + directory: directory.into(), + })) + } + + pub(super) async fn bearer(&self) -> Result { + let mut token = self.token.lock().await; + if token.expiry > Utc::now() + chrono::Duration::minutes(5) { + return Ok(token.value.clone()); + } + if let Ok(updated) = token_files(&self.directory) + && updated.expiry > token.expiry + { + *token = updated; + } + if token.expiry <= Utc::now() { + return Err("Private SRE Kubernetes credential expired; no ambient fallback".into()); + } + let path = format!( + "/api/v1/namespaces/{}/serviceaccounts/sre-api-router/token", + self.config.runtime_namespace + ); + let response=self.client.post(format!("{}{}",self.config.kube_url.trim_end_matches('/'),path)) + .bearer_auth(&token.value).json(&json!({ + "apiVersion":"authentication.k8s.io/v1","kind":"TokenRequest", + "spec":{"audiences":[],"expirationSeconds":3600,"boundObjectRef":{ + "apiVersion":"v1","kind":"Secret","name":"sre-api-router-identity","uid":self.config.secret_uid}}, + })).send().await.map_err(|_|"SRE token renewal transport failure")?; + if !response.status().is_success() { + return Err("SRE token renewal was denied; no ambient fallback".into()); + } + let value: Value = response + .json() + .await + .map_err(|_| "SRE token renewal response invalid")?; + let new_token = value["status"]["token"] + .as_str() + .filter(|s| !s.is_empty()) + .ok_or("SRE token renewal omitted token")?; + let expiry = value["status"]["expirationTimestamp"] + .as_str() + .ok_or("SRE token renewal omitted expiry")?; + let expiry = DateTime::parse_from_rfc3339(expiry) + .map_err(|_| "SRE token renewal expiry invalid")? + .with_timezone(&Utc); + if expiry <= Utc::now() + chrono::Duration::minutes(5) { + return Err("SRE token renewal returned an unusable lifetime".into()); + } + *token = Token { + value: new_token.into(), + expiry, + }; + Ok(token.value.clone()) + } + + async fn metadata_json(&self, path: &str) -> Result { + let response = self + .client + .get(format!( + "{}{}", + self.config.kube_url.trim_end_matches('/'), + path + )) + .bearer_auth(self.bearer().await?) + .header("accept", "application/json") + .send() + .await + .map_err(|_| "SRE authority read failed")?; + if !response.status().is_success() { + return Err("SRE authority read denied".into()); + } + response + .json() + .await + .map_err(|_| "SRE authority response invalid".into()) + } + + pub(super) async fn authorize(&self) -> Result<(), String> { + let reg = self + .metadata_json("/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical") + .await?; + if reg["metadata"]["uid"] != self.config.registration_uid + || !reg["metadata"]["deletionTimestamp"].is_null() + || reg["spec"]["enabled"] != true + || reg["status"]["phase"] != "Ready" + || reg["status"]["observedGeneration"] != reg["metadata"]["generation"] + || reg["status"]["privacyEpoch"] != self.config.privacy_epoch + || reg["status"]["legacySecretAccessDenied"] != true + || reg["status"]["privacyRevision"] != crate::sre_privacy::REVISION + || reg["spec"]["sandbox"]["uid"] != self.config.source.uid + || reg["spec"]["sandbox"]["namespace"] != self.config.source.namespace + || reg["spec"]["runtimeNamespace"]["uid"] != self.config.namespace_uid + { + return Err("SRE authority is no longer current".into()); + } + let namespace = self + .metadata_json(&format!( + "/api/v1/namespaces/{}", + self.config.runtime_namespace + )) + .await?; + let annotations = &namespace["metadata"]["annotations"]; + if namespace["metadata"]["uid"] != self.config.namespace_uid + || !namespace["metadata"]["deletionTimestamp"].is_null() + || namespace["metadata"]["ownerReferences"] + .as_array() + .is_some_and(|refs| !refs.is_empty()) + || annotations["kars.azure.com/namespace-claim-version"] != "v1" + || annotations["kars.azure.com/sandbox-namespace"] != self.config.source.namespace + || annotations["kars.azure.com/sandbox-name"] != self.config.source.name + || annotations["kars.azure.com/sandbox-uid"] != self.config.source.uid + || !annotations["kars.azure.com/namespace-prestage"].is_null() + { + return Err("SRE runtime namespace ownership changed".into()); + } + let sandbox = self + .metadata_json(&format!( + "/apis/kars.azure.com/v1alpha1/namespaces/{}/karssandboxes/{}", + self.config.source.namespace, self.config.source.name + )) + .await?; + if sandbox["metadata"]["uid"] != self.config.source.uid + || !sandbox["metadata"]["deletionTimestamp"].is_null() + || sandbox["metadata"]["annotations"]["kars.azure.com/namespace-uid"] + != self.config.namespace_uid + { + return Err("SRE source identity changed".into()); + } + let sa = self + .metadata_json(&format!( + "/api/v1/namespaces/{}/serviceaccounts/sre-api-router", + self.config.runtime_namespace + )) + .await?; + if sa["metadata"]["uid"] != self.config.service_account_uid + || !sa["metadata"]["deletionTimestamp"].is_null() + { + return Err("Private SRE ServiceAccount was replaced".into()); + } + self.verify_privacy().await?; + Ok(()) + } + + async fn verify_privacy(&self) -> Result<(), String> { + for review in crate::sre_privacy::secret_access_reviews(&self.config.runtime_namespace) { + let response = self + .client + .post(format!( + "{}/apis/authorization.k8s.io/v1/subjectaccessreviews", + self.config.kube_url.trim_end_matches('/') + )) + .bearer_auth(self.bearer().await?) + .json(&review) + .send() + .await + .map_err(|_| "SRE privacy authorization transport failure")?; + if !response.status().is_success() { + return Err("SRE privacy authorization review denied".into()); + } + let response: Value = response + .json() + .await + .map_err(|_| "SRE privacy authorization response invalid")?; + crate::sre_privacy::require_denial(&response)?; + } + let response = self + .client + .get(format!( + "{}/api/v1/namespaces/{}/secrets", + self.config.kube_url.trim_end_matches('/'), + self.config.runtime_namespace + )) + .bearer_auth(self.bearer().await?) + .header( + "accept", + "application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1", + ) + .send() + .await + .map_err(|_| "SRE credential metadata inventory failed")?; + if !response.status().is_success() { + return Err("SRE credential metadata inventory denied".into()); + } + let metadata: Value = response + .json() + .await + .map_err(|_| "SRE credential metadata response invalid")?; + crate::sre_privacy::reject_legacy_aliases( + &metadata, + &[self.config.service_account_uid.as_str()], + )?; + Ok(()) + } + + pub(super) async fn forward( + &self, + method: reqwest::Method, + path: &str, + body: Option, + logs: bool, + ) -> Result { + self.authorize().await?; + let mut request = self + .client + .request( + method, + format!("{}{}", self.config.kube_url.trim_end_matches('/'), path), + ) + .bearer_auth(self.bearer().await?) + .header( + "accept", + if logs { + "text/plain" + } else { + "application/json" + }, + ); + if let Some(body) = body { + request = request.json(&body); + } + request + .send() + .await + .map_err(|_| "Kubernetes diagnostic transport failure".into()) + } + + pub(super) fn renew_in_background(self: &Arc) { + let backend = self.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(60)).await; + if backend.bearer().await.is_err() { + tracing::warn!("Private SRE token renewal unavailable; requests fail closed"); + } + } + }); + } +} diff --git a/inference-router/src/sre_proxy/mod.rs b/inference-router/src/sre_proxy/mod.rs new file mode 100644 index 000000000..c539361a6 --- /dev/null +++ b/inference-router/src/sre_proxy/mod.rs @@ -0,0 +1,320 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Agent-safe HTTPS facade for the existing Hermes Kubernetes client. + +mod backend; +mod policy; +#[cfg(test)] +mod tests; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::{DefaultBodyLimit, State}, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + routing::get, +}; +use backend::Backend; +use futures::StreamExt; +use policy::Route; +use std::{ + io::{self, BufReader}, + net::SocketAddr, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::Semaphore, +}; +use tokio_rustls::{TlsAcceptor, server::TlsStream}; + +const DIRECTORY: &str = "/etc/kars/sre-api"; +pub const PORT: u16 = 9446; +const MAX_RESPONSE: usize = 8 * 1024 * 1024; + +#[derive(Clone)] +struct Proxy { + backend: Arc, + token: Arc, + capacity: Arc, +} + +fn error(status: StatusCode, message: &str) -> Response { + ( + status, + axum::Json(serde_json::json!({ + "apiVersion":"v1","kind":"Status","status":"Failure", + "code":status.as_u16(),"reason":"SREProxyDenied","message":message, + })), + ) + .into_response() +} + +async fn ready(State(proxy): State) -> Response { + let Ok(_permit) = proxy.capacity.try_acquire() else { + return error( + StatusCode::TOO_MANY_REQUESTS, + "SRE proxy capacity is exhausted", + ); + }; + match proxy.backend.authorize().await { + Ok(()) => ( + StatusCode::OK, + axum::Json(serde_json::json!({"ready":true})), + ) + .into_response(), + Err(_) => error( + StatusCode::SERVICE_UNAVAILABLE, + "SRE authority is not ready", + ), + } +} + +async fn forward( + State(proxy): State, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + if headers.get_all("authorization").iter().count() != 1 + || headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + != Some(proxy.token.as_ref()) + { + return error( + StatusCode::UNAUTHORIZED, + "An SRE proxy credential is required", + ); + } + let route = match policy::route(&method, &uri) { + Ok(route) => route, + Err(message) => return error(StatusCode::FORBIDDEN, message), + }; + if headers.contains_key("upgrade") + || headers + .get("accept") + .and_then(|value| value.to_str().ok()) + .is_some_and(|accept| { + !(matches!(accept, "application/json" | "*/*") + || route == Route::Logs && accept == "text/plain") + }) + { + return error( + StatusCode::NOT_ACCEPTABLE, + "Only the bounded JSON/log SRE API is available", + ); + } + let request_body = if route == Route::Proposal { + if headers.get("content-type").and_then(|v| v.to_str().ok()) != Some("application/json") { + return error( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "SRE proposals require application/json", + ); + } + let value = match serde_json::from_slice::(&body) { + Ok(value) => value, + Err(_) => return error(StatusCode::BAD_REQUEST, "Proposal JSON is invalid"), + }; + match policy::proposal(&value) { + Ok(value) => Some(value), + Err(message) => return error(StatusCode::FORBIDDEN, message), + } + } else { + if !body.is_empty() { + return error( + StatusCode::BAD_REQUEST, + "Diagnostic GET bodies are not allowed", + ); + } + None + }; + let Ok(_permit) = proxy.capacity.try_acquire() else { + return error( + StatusCode::TOO_MANY_REQUESTS, + "SRE proxy capacity is exhausted", + ); + }; + let path = uri + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + let response = match proxy + .backend + .forward(method, path, request_body, route == Route::Logs) + .await + { + Ok(response) => response, + Err(_) => { + return error( + StatusCode::SERVICE_UNAVAILABLE, + "SRE Kubernetes authority or transport is unavailable", + ); + } + }; + let status = response.status(); + if !status.is_success() { + return error( + status, + "Kubernetes rejected the diagnostic/proposal request", + ); + } + let mut stream = response.bytes_stream(); + let mut bytes = Vec::new(); + let limit = if route == Route::Logs { + 262144 + } else { + MAX_RESPONSE + }; + while let Some(chunk) = stream.next().await { + let Ok(chunk) = chunk else { + return error( + StatusCode::BAD_GATEWAY, + "Kubernetes response was incomplete", + ); + }; + if bytes.len() + chunk.len() > limit { + return error( + StatusCode::BAD_GATEWAY, + "Kubernetes response exceeds the SRE diagnostic limit", + ); + } + bytes.extend_from_slice(&chunk); + } + if route == Route::Logs { + return ( + status, + [("content-type", "text/plain; charset=utf-8")], + Body::from(bytes), + ) + .into_response(); + } + let value = match serde_json::from_slice::(&bytes) { + Ok(value) => value, + Err(_) => return error(StatusCode::BAD_GATEWAY, "Kubernetes response was not JSON"), + }; + let value = if route == Route::Secrets { + match policy::secret_projection(&value) { + Ok(value) => value, + Err(message) => return error(StatusCode::BAD_GATEWAY, message), + } + } else { + value + }; + (status, axum::Json(value)).into_response() +} + +fn app(proxy: Proxy) -> Router { + Router::new() + .route("/readyz", get(ready)) + .fallback(forward) + .layer(DefaultBodyLimit::max(65_536)) + .with_state(proxy) +} + +struct Listener { + tcp: TcpListener, + tls: TlsAcceptor, +} + +impl axum::serve::Listener for Listener { + type Io = TlsStream; + type Addr = SocketAddr; + async fn accept(&mut self) -> (Self::Io, Self::Addr) { + loop { + match self.tcp.accept().await { + Ok((stream, address)) => { + if let Ok(Ok(stream)) = tokio::time::timeout( + std::time::Duration::from_secs(3), + self.tls.accept(stream), + ) + .await + { + return (stream, address); + } + } + Err(_) => tokio::time::sleep(std::time::Duration::from_millis(100)).await, + } + } + } + fn local_addr(&self) -> io::Result { + self.tcp.local_addr() + } +} + +fn tls(directory: &Path) -> Result { + let certificates = std::fs::File::open(directory.join("server-cert.pem")) + .map_err(|_| "SRE TLS certificate unavailable")?; + let certificates = rustls_pemfile::certs(&mut BufReader::new(certificates)) + .collect::, _>>() + .map_err(|_| "SRE TLS certificate invalid")?; + let key = std::fs::File::open(directory.join("server-key.pem")) + .map_err(|_| "SRE TLS key unavailable")?; + let key = rustls_pemfile::private_key(&mut BufReader::new(key)) + .map_err(|_| "SRE TLS key invalid")? + .ok_or("SRE TLS private key missing")?; + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, key) + .map_err(|_| "SRE TLS certificate/key mismatch")?; + Ok(TlsAcceptor::from(Arc::new(config))) +} + +pub async fn start() -> Result>, String> { + if std::env::var("KARS_SRE_API_ENABLED").as_deref() != Ok("true") { + return Ok(None); + } + let directory = PathBuf::from(DIRECTORY); + let backend = Backend::load(&directory)?; + let token = std::fs::read_to_string(directory.join("agent-token")) + .map_err(|_| "SRE proxy credential unavailable")?; + if token.trim().len() != 64 || !token.trim().bytes().all(|b| b.is_ascii_alphanumeric()) { + return Err("SRE proxy credential invalid".into()); + } + let listener = Listener { + tcp: TcpListener::bind(("127.0.0.1", PORT)) + .await + .map_err(|_| "SRE loopback TLS listener unavailable")?, + tls: tls(&directory)?, + }; + backend.renew_in_background(); + let proxy = Proxy { + backend, + token: Arc::from(token.trim()), + capacity: Arc::new(Semaphore::new(16)), + }; + let router = app(proxy); + Ok(Some(tokio::spawn(async move { + if axum::serve(listener, router).await.is_err() { + tracing::error!("SRE TLS proxy stopped; router must restart"); + std::process::exit(1); + } + }))) +} + +pub async fn readiness_probe() -> bool { + let Ok(ca) = std::fs::read(Path::new(DIRECTORY).join("agent-ca.crt")) else { + return false; + }; + let Ok(ca) = reqwest::Certificate::from_pem(&ca) else { + return false; + }; + let Ok(client) = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(ca) + .timeout(std::time::Duration::from_secs(3)) + .build() + else { + return false; + }; + client + .get(format!("https://127.0.0.1:{PORT}/readyz")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) +} diff --git a/inference-router/src/sre_proxy/policy.rs b/inference-router/src/sre_proxy/policy.rs new file mode 100644 index 000000000..73b2a8c07 --- /dev/null +++ b/inference-router/src/sre_proxy/policy.rs @@ -0,0 +1,409 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Closed Kubernetes route/query/media contract for an untrusted SRE agent. + +use axum::http::{Method, Uri}; +use serde_json::{Value, json}; +use std::collections::BTreeSet; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Route { + Json, + Secrets, + Logs, + Proposal, +} + +fn label(value: &str) -> bool { + !value.is_empty() + && value.len() <= 253 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b'_')) + && value != "." + && value != ".." +} + +fn resource(group: &str, version: &str, name: &str) -> bool { + match (group, version) { + ("", "v1") => [ + "pods", + "services", + "endpoints", + "events", + "configmaps", + "secrets", + "serviceaccounts", + "resourcequotas", + "limitranges", + "nodes", + "namespaces", + ] + .contains(&name), + ("apps", "v1") => { + ["deployments", "replicasets", "statefulsets", "daemonsets"].contains(&name) + } + ("batch", "v1") => ["jobs", "cronjobs"].contains(&name), + ("networking.k8s.io", "v1") => ["networkpolicies", "ingresses"].contains(&name), + ("discovery.k8s.io", "v1") => name == "endpointslices", + ("events.k8s.io", "v1") => name == "events", + ("metrics.k8s.io", "v1beta1") => ["pods", "nodes"].contains(&name), + ("apiextensions.k8s.io", "v1") => name == "customresourcedefinitions", + ("rbac.authorization.k8s.io", "v1") => [ + "roles", + "rolebindings", + "clusterroles", + "clusterrolebindings", + ] + .contains(&name), + ("kars.azure.com", "v1alpha1") => [ + "karssandboxes", + "inferencepolicies", + "toolpolicies", + "mcpservers", + "karsmemories", + "karsevals", + "karstasks", + "karsteams", + "karsprofiles", + "karsskills", + "karsreceipts", + "karsapprovals", + "egressapprovals", + "karssreactions", + "karsauthconfigs", + "trustgraphs", + "a2aagents", + "karspairings", + ] + .contains(&name), + _ => false, + } +} + +pub(super) fn route(method: &Method, uri: &Uri) -> Result { + let path = uri.path(); + if uri.scheme().is_some() + || uri.authority().is_some() + || path.contains('%') + || path.contains('\\') + || path.contains("//") + || path.ends_with('/') + || path.bytes().any(|b| b.is_ascii_control()) + { + return Err("Noncanonical Kubernetes path"); + } + let parts: Vec<_> = path.split('/').skip(1).collect(); + if parts.iter().any(|part| !label(part)) { + return Err("Invalid Kubernetes path component"); + } + let (group, version, tail) = match parts.as_slice() { + ["api", version, tail @ ..] => ("", *version, tail), + ["apis", group, version, tail @ ..] => (*group, *version, tail), + _ => return Err("Kubernetes discovery/proxy paths are not part of the SRE contract"), + }; + let (namespace, kind, name, subresource) = match tail { + [kind] => (None, *kind, None, None), + ["namespaces", ns, kind] => (Some(*ns), *kind, None, None), + ["namespaces", ns, kind, name] => (Some(*ns), *kind, Some(*name), None), + ["namespaces", ns, kind, name, sub] => (Some(*ns), *kind, Some(*name), Some(*sub)), + [kind, name] + if [ + "nodes", + "namespaces", + "customresourcedefinitions", + "clusterroles", + "clusterrolebindings", + ] + .contains(kind) => + { + (None, *kind, Some(*name), None) + } + _ => return Err("Kubernetes subresource is not allowed"), + }; + if !resource(group, version, kind) { + return Err("Kubernetes resource is not allowed"); + } + if method == Method::POST { + if group == "kars.azure.com" + && version == "v1alpha1" + && namespace == Some("kars-sre") + && kind == "karssreactions" + && name.is_none() + && subresource.is_none() + && uri.query().is_none() + { + return Ok(Route::Proposal); + } + return Err("Only Pending SRE proposal creation is permitted"); + } + if method != Method::GET { + return Err("Kubernetes writes are not permitted"); + } + let result = match subresource { + Some("log") if group.is_empty() && kind == "pods" => Route::Logs, + Some(_) => return Err("Kubernetes exec/proxy/token subresources are not permitted"), + None if group.is_empty() && kind == "secrets" => Route::Secrets, + None => Route::Json, + }; + validate_query(uri.query(), result)?; + Ok(result) +} + +fn validate_query(query: Option<&str>, route: Route) -> Result<(), &'static str> { + let Some(query) = query else { return Ok(()) }; + let mut keys = BTreeSet::new(); + let mut parsed = reqwest::Url::parse("https://localhost/").expect("static URL"); + parsed.set_query(Some(query)); + for (key, value) in parsed.query_pairs() { + if !keys.insert(key.to_string()) + || value.len() > 2048 + || value.chars().any(char::is_control) + { + return Err("Invalid or duplicate Kubernetes query parameter"); + } + match (route, key.as_ref()) { + (Route::Logs, "container") if label(&value) => {} + (Route::Logs, "tailLines") if value.parse::().is_ok_and(|n| n <= 1000) => {} + (Route::Logs, "limitBytes") if value.parse::().is_ok_and(|n| n <= 262144) => {} + (Route::Logs, "sinceSeconds") if value.parse::().is_ok_and(|n| n <= 86400) => {} + (Route::Logs, "timestamps" | "previous") + if matches!(value.as_ref(), "true" | "false") => {} + (Route::Json | Route::Secrets, "limit") + if value.parse::().is_ok_and(|n| n > 0 && n <= 500) => {} + (Route::Json | Route::Secrets, "labelSelector" | "fieldSelector") => {} + (Route::Json | Route::Secrets, "continue") if value.len() <= 1024 => {} + _ => return Err("Kubernetes query parameter is not permitted"), + } + } + Ok(()) +} + +fn secret(value: &Value) -> Result { + if value["kind"] != "Secret" || !value["metadata"].is_object() { + return Err("Malformed Secret response"); + } + let mut metadata = serde_json::Map::new(); + for key in [ + "name", + "namespace", + "uid", + "resourceVersion", + "creationTimestamp", + "deletionTimestamp", + ] { + if let Some(value) = value["metadata"].get(key) { + metadata.insert(key.into(), value.clone()); + } + } + let keys: serde_json::Map = value + .get("data") + .and_then(Value::as_object) + .into_iter() + .flatten() + .map(|(key, _)| (key.clone(), Value::String(String::new()))) + .collect(); + Ok( + json!({"apiVersion":"v1","kind":"Secret","metadata":metadata, + "type":value.get("type").cloned().unwrap_or(Value::Null),"data":keys}), + ) +} + +pub(super) fn secret_projection(value: &Value) -> Result { + if value["kind"] == "Secret" { + return secret(value); + } + if value["kind"] != "SecretList" { + return Err("Unexpected Secret response kind"); + } + let items = value["items"].as_array().ok_or("Malformed Secret list")?; + let items = items.iter().map(secret).collect::, _>>()?; + Ok(json!({"apiVersion":"v1","kind":"SecretList", + "metadata":{"resourceVersion":value["metadata"]["resourceVersion"],"continue":value["metadata"]["continue"]}, + "items":items})) +} + +pub(super) fn proposal(value: &Value) -> Result { + if value["apiVersion"] != "kars.azure.com/v1alpha1" + || value["kind"] != "KarsSREAction" + || value.get("status").is_some() + || !value["spec"].is_object() + { + return Err("Invalid SRE proposal"); + } + let metadata = value["metadata"] + .as_object() + .ok_or("Proposal metadata missing")?; + if metadata + .keys() + .any(|key| !["name", "generateName", "namespace", "labels"].contains(&key.as_str())) + || metadata.get("namespace").is_some_and(|ns| ns != "kars-sre") + || !metadata + .get("name") + .or_else(|| metadata.get("generateName")) + .and_then(Value::as_str) + .is_some_and(label) + { + return Err("Proposal identity is invalid"); + } + if let Some(labels) = metadata.get("labels") { + let labels = labels + .as_object() + .ok_or("Proposal labels must be an object")?; + if labels.iter().any(|(key, label)| match key.as_str() { + "app.kubernetes.io/component" => label != "sre", + "kars.azure.com/sre-action-type" => label != &value["spec"]["action"]["type"], + _ => true, + }) { + return Err("Only the existing SRE diagnostic labels are permitted"); + } + } + let spec = value["spec"].as_object().unwrap(); + if spec.keys().any(|key| { + !["action", "rationale", "diagnosis", "approval", "ttlMinutes"].contains(&key.as_str()) + }) || spec + .get("approval") + .is_some_and(|approval| approval != &json!({"state":"Pending"})) + || ![ + "DeleteResourceQuota", + "PatchDeploymentImage", + "ScaleDeployment", + "RolloutRestart", + "DeletePod", + ] + .contains(&value["spec"]["action"]["type"].as_str().unwrap_or_default()) + { + return Err("SRE proposals cannot grant approval or change unsupported authority"); + } + let mut result = value.clone(); + result["metadata"]["namespace"] = "kars-sre".into(); + result["spec"]["approval"] = json!({"state":"Pending"}); + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diagnostic_contract_keeps_reads_logs_metrics_and_only_pending_proposals() { + for path in [ + "/api/v1/namespaces", + "/api/v1/namespaces/kars-sre/pods", + "/api/v1/namespaces/kars-sre/pods/sre-123/log?tailLines=100×tamps=true", + "/apis/metrics.k8s.io/v1beta1/nodes", + "/apis/kars.azure.com/v1alpha1/karssandboxes", + "/apis/apps/v1/namespaces/kars-system/deployments/kars-controller", + ] { + assert!( + route(&Method::GET, &path.parse().unwrap()).is_ok(), + "{path}" + ); + } + assert_eq!( + route( + &Method::POST, + &"/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions" + .parse() + .unwrap() + ) + .unwrap(), + Route::Proposal + ); + } + + #[test] + fn path_query_and_write_escapes_fail_closed() { + for path in [ + "/api/v1/namespaces/kars-sre/secrets?watch=true", + "/api/v1/namespaces/kars-sre/pods/sre/log?follow=true", + "/api/v1/namespaces/kars-sre/pods/sre/log?tailLines=1&tailLines=2", + "/api/v1/namespaces/kars-sre/pods/sre/exec", + "/api/v1/namespaces/kars-sre/pods/sre/proxy", + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token", + "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical", + "/api/v1/namespaces/kars-sre/secrets/../pods", + "/api/v1/namespaces/kars-sre/%73ecrets", + "/api/v1/namespaces/kars-sre/pods%2fsre%2fproxy", + "/api//v1/namespaces/kars-sre/secrets", + "/api/v1/namespaces/kars-sre/secrets/", + "https://kubernetes.default.svc/api/v1/secrets", + ] { + assert!( + route(&Method::GET, &path.parse().unwrap()).is_err(), + "{path}" + ); + } + for method in [Method::PUT, Method::PATCH, Method::DELETE, Method::CONNECT] { + assert!( + route( + &method, + &"/api/v1/namespaces/kars-sre/secrets/secret" + .parse() + .unwrap() + ) + .is_err() + ); + } + assert!( + route( + &Method::POST, + &"/apis/kars.azure.com/v1alpha1/namespaces/other/karssreactions" + .parse() + .unwrap() + ) + .is_err() + ); + } + + #[test] + fn secrets_and_lists_keep_key_names_without_any_value_or_annotation_copy() { + let secret = json!({"apiVersion":"v1","kind":"Secret","type":"Opaque", + "metadata":{"name":"router-services-admin","namespace":"kars-test","uid":"uid", + "annotations":{"kubectl.kubernetes.io/last-applied-configuration":"PRIVATE_VALUE"}, + "labels":{"copied":"PRIVATE_VALUE"},"managedFields":[{"copy":"PRIVATE_VALUE"}]}, + "data":{"control-token":"PRIVATE_VALUE"},"stringData":{"copy":"PRIVATE_VALUE"}}); + for input in [ + secret.clone(), + json!({"kind":"SecretList","metadata":{},"items":[secret]}), + ] { + let output = secret_projection(&input).unwrap(); + let encoded = serde_json::to_string(&output).unwrap(); + assert!(!encoded.contains("PRIVATE_VALUE")); + assert!(!encoded.contains("annotations")); + assert!(!encoded.contains("stringData")); + assert!(!encoded.contains("managedFields")); + assert!(encoded.contains("control-token")); + } + } + + #[test] + fn proposals_cannot_self_approve_or_inject_status_ownership_or_extra_fields() { + let base = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction", + "metadata":{"name":"proposal","namespace":"kars-sre"}, + "spec":{"action":{"type":"RolloutRestart","params":{"namespace":"kars-test","name":"app"}}}}); + assert_eq!( + proposal(&base).unwrap()["spec"]["approval"], + json!({"state":"Pending"}) + ); + for (pointer, value) in [ + ("/spec/approval", json!({"state":"Approved"})), + ("/metadata/namespace", json!("other")), + ] { + let mut changed = base.clone(); + if pointer == "/spec/approval" { + changed["spec"]["approval"] = value; + } else { + changed["metadata"]["namespace"] = value; + } + assert!(proposal(&changed).is_err()); + } + let mut changed = base.clone(); + changed["status"] = json!({"phase":"Approved"}); + assert!(proposal(&changed).is_err()); + let mut changed = base; + changed["metadata"]["ownerReferences"] = json!([]); + assert!(proposal(&changed).is_err()); + } +} diff --git a/inference-router/src/sre_proxy/tests.rs b/inference-router/src/sre_proxy/tests.rs new file mode 100644 index 000000000..28c257253 --- /dev/null +++ b/inference-router/src/sre_proxy/tests.rs @@ -0,0 +1,380 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::*; +use backend::{Config, Source}; +use chrono::Utc; +use serde_json::json; +use std::sync::Mutex; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const PRIVATE_VALUE: &str = "PRIVATE_OPERATOR_CONTROL_VALUE"; + +struct Fixture { + _upstream: MockServer, + directory: tempfile::TempDir, + task: tokio::task::JoinHandle<()>, + backend: Arc, + client: reqwest::Client, + url: String, + token: String, + privacy: Arc>, +} + +#[derive(Default)] +struct PrivacyState { + aliases: Vec, + watch_allowed: bool, + prior_revision: bool, + metadata_error: Option, + calls: Vec<(String, String, serde_json::Value)>, +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn fixture() -> Fixture { + let upstream = MockServer::start().await; + let privacy = Arc::new(Mutex::new(PrivacyState::default())); + let observed = privacy.clone(); + Mock::given(|_:&wiremock::Request|true).respond_with(move |request:&wiremock::Request| { + if request.headers.get("authorization").and_then(|v|v.to_str().ok())!=Some("Bearer private-kubernetes-token") { + return ResponseTemplate::new(401).set_body_json(json!({"kind":"Status","code":401})); + } + let path=request.url.path(); + let mut state=observed.lock().unwrap(); + let body:serde_json::Value=request.body_json().unwrap_or_default(); + state.calls.push((request.method.to_string(),path.into(),body.clone())); + let value=match path { + "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical" => json!({ + "metadata":{"uid":"registration","generation":1}, + "spec":{"enabled":true,"sandbox":{"namespace":"kars-system","uid":"source"},"runtimeNamespace":{"uid":"namespace"}}, + "status":{"phase":"Ready","observedGeneration":1,"privacyEpoch":"epoch","legacySecretAccessDenied":true, + "privacyRevision":if state.prior_revision {None} else {Some(crate::sre_privacy::REVISION)}}, + }), + "/api/v1/namespaces/kars-sre" => json!({"metadata":{"uid":"namespace","annotations":{ + "kars.azure.com/namespace-claim-version":"v1","kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre","kars.azure.com/sandbox-uid":"source"}}}), + "/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/sre" => + json!({"metadata":{"uid":"source","annotations":{"kars.azure.com/namespace-uid":"namespace"}}}), + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router" => json!({"metadata":{"uid":"router-sa"}}), + "/apis/authorization.k8s.io/v1/subjectaccessreviews" if request.method=="POST" => + json!({"status":{"allowed":state.watch_allowed && body["spec"]["resourceAttributes"]["verb"]=="watch"}}), + "/api/v1/namespaces/kars-sre/secrets" => { + assert!(request.headers["accept"].to_str().unwrap().contains("PartialObjectMetadataList")); + if let Some(code)=state.metadata_error { + return ResponseTemplate::new(code).set_body_json(json!({"message":PRIVATE_VALUE})); + } + json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{},"items":state.aliases}) + } + "/api/v1/namespaces/kars-demo/secrets/router-services-admin" => secret(), + "/api/v1/secrets" => json!({"apiVersion":"v1","kind":"SecretList","metadata":{},"items":[secret()]}), + "/api/v1/namespaces/kars-demo/pods/app/log" => return ResponseTemplate::new(200).set_body_raw("legitimate pod log\n","text/plain"), + "/apis/metrics.k8s.io/v1beta1/nodes" => json!({"kind":"NodeMetricsList","items":[]}), + "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions" if request.method=="POST" => { + let body:serde_json::Value=request.body_json().unwrap(); + assert_eq!(body["spec"]["approval"],json!({"state":"Pending"})); + return ResponseTemplate::new(201).set_body_json(body); + } + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token" if request.method=="POST" => + json!({"status":{"token":"private-kubernetes-token","expirationTimestamp":(Utc::now()+chrono::Duration::hours(1)).to_rfc3339()}}), + _ => return ResponseTemplate::new(404).set_body_json(json!({"kind":"Status","code":404})), + }; + ResponseTemplate::new(200).set_body_json(value) + }).mount(&upstream).await; + let directory = tempfile::tempdir_in(".").unwrap(); + let key = rcgen::KeyPair::generate().unwrap(); + let mut params = + rcgen::CertificateParams::new(vec!["localhost".into(), "127.0.0.1".into()]).unwrap(); + params + .distinguished_name + .push(rcgen::DnType::CommonName, "SRE compatibility test"); + let cert = params.self_signed(&key).unwrap(); + std::fs::write(directory.path().join("server-cert.pem"), cert.pem()).unwrap(); + std::fs::write(directory.path().join("server-key.pem"), key.serialize_pem()).unwrap(); + std::fs::write(directory.path().join("ca.crt"), cert.pem()).unwrap(); + let token = "a".repeat(64); + std::fs::write(directory.path().join("token"), &token).unwrap(); + std::fs::write(directory.path().join("namespace"), "kars-sre").unwrap(); + let backend = Backend::for_test( + Config { + schema: "kars.azure.com/sre-api/v1".into(), + kube_url: upstream.uri(), + registration_uid: "registration".into(), + privacy_epoch: "epoch".into(), + source: Source { + namespace: "kars-system".into(), + name: "sre".into(), + uid: "source".into(), + }, + runtime_namespace: "kars-sre".into(), + namespace_uid: "namespace".into(), + service_account_uid: "router-sa".into(), + secret_uid: "secret".into(), + }, + directory.path().into(), + Utc::now() + chrono::Duration::hours(1), + ); + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let tcp = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let url = format!("https://{}", tcp.local_addr().unwrap()); + let listener = Listener { + tcp, + tls: tls(directory.path()).unwrap(), + }; + let proxy = Proxy { + backend: backend.clone(), + token: Arc::from(token.as_str()), + capacity: Arc::new(Semaphore::new(16)), + }; + let task = tokio::spawn(async move { axum::serve(listener, app(proxy)).await.unwrap() }); + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(reqwest::Certificate::from_pem(cert.pem().as_bytes()).unwrap()) + .build() + .unwrap(); + Fixture { + _upstream: upstream, + directory, + task, + backend, + client, + url, + token, + privacy, + } +} + +#[tokio::test] +async fn prior_ready_watch_only_access_aliases_and_inventory_errors_fail_closed() { + for case in [ + "prior-ready", + "watch-only", + "alias-name", + "alias-uid", + "inventory-error", + ] { + let f = fixture().await; + { + let mut state = f.privacy.lock().unwrap(); + match case { + "prior-ready"=>state.prior_revision=true, + "watch-only"=>state.watch_allowed=true, + "inventory-error"=>state.metadata_error=Some(403), + _=>state.aliases.push(json!({"metadata":{"name":"arbitrary-token-alias","uid":"alias","resourceVersion":"1", + "annotations":if case=="alias-name" { + json!({"kubernetes.io/service-account.name":"sre-api-router"}) + } else {json!({"kubernetes.io/service-account.name":"renamed","kubernetes.io/service-account.uid":"router-sa"})}}})), + } + } + let response = f + .client + .get(format!( + "{}/api/v1/namespaces/kars-demo/secrets/router-services-admin", + f.url + )) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{case}"); + assert!(!response.text().await.unwrap().contains(PRIVATE_VALUE)); + assert!( + f.privacy + .lock() + .unwrap() + .calls + .iter() + .all(|(_, path, _)| path + != "/api/v1/namespaces/kars-demo/secrets/router-services-admin"), + "{case}" + ); + } +} + +fn secret() -> serde_json::Value { + json!({"apiVersion":"v1","kind":"Secret","type":"Opaque", + "metadata":{"name":"router-services-admin","namespace":"kars-demo", + "annotations":{"kubectl.kubernetes.io/last-applied-configuration":PRIVATE_VALUE}, + "labels":{"copy":PRIVATE_VALUE},"managedFields":[{"copy":PRIVATE_VALUE}]}, + "data":{"control-token":PRIVATE_VALUE},"stringData":{"copy":PRIVATE_VALUE}}) +} + +#[tokio::test] +async fn agent_credential_cannot_read_control_material_directly_or_through_tls_proxy() { + let f = fixture().await; + let direct = reqwest::Client::new() + .get(format!( + "{}/api/v1/namespaces/kars-demo/secrets/router-services-admin", + f.backend.config.kube_url + )) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(direct.status(), StatusCode::UNAUTHORIZED); + for path in [ + "/api/v1/namespaces/kars-demo/secrets/router-services-admin", + "/api/v1/secrets", + ] { + let response = f + .client + .get(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.text().await.unwrap(); + assert!(body.contains("control-token")); + assert!(!body.contains(PRIVATE_VALUE)); + assert!(!body.contains("annotations")); + assert!(!body.contains("stringData")); + } +} + +#[tokio::test] +async fn tls_proxy_preserves_logs_metrics_and_pending_proposals_but_rejects_escapes() { + let f = fixture().await; + let log = f + .client + .get(format!( + "{}/api/v1/namespaces/kars-demo/pods/app/log?tailLines=100", + f.url + )) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(log.text().await.unwrap(), "legitimate pod log\n"); + assert!( + f.client + .get(format!("{}/apis/metrics.k8s.io/v1beta1/nodes", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap() + .status() + .is_success() + ); + let path = "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions"; + let mut proposal = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction", + "metadata":{"name":"proposal"},"spec":{"action":{"type":"RolloutRestart","params":{}}}}); + assert_eq!( + f.client + .post(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .json(&proposal) + .send() + .await + .unwrap() + .status(), + StatusCode::CREATED + ); + proposal["spec"]["approval"] = json!({"state":"Approved"}); + assert_eq!( + f.client + .post(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .json(&proposal) + .send() + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + for path in [ + "/api/v1/%73ecrets", + "/api/v1/secrets?watch=true", + "/api/v1/namespaces/kars-sre/pods/sre/proxy", + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token", + ] { + assert_eq!( + f.client + .get(format!("{}{path}", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + } + assert_eq!( + f.client + .get(format!("{}/api/v1/secrets", f.url)) + .bearer_auth(&f.token) + .header("accept", "application/vnd.kubernetes.protobuf") + .send() + .await + .unwrap() + .status(), + StatusCode::NOT_ACCEPTABLE + ); +} + +#[tokio::test] +async fn private_identity_renews_without_an_ambient_credential_fallback() { + let f = fixture().await; + let backend = Backend::for_test( + f.backend.config.clone(), + f.directory.path().into(), + Utc::now() + chrono::Duration::minutes(2), + ); + assert_eq!(backend.bearer().await.unwrap(), "private-kubernetes-token"); + assert!(backend.authorize().await.is_ok()); + let expired = Backend::for_test( + f.backend.config.clone(), + f.directory.path().into(), + Utc::now() - chrono::Duration::seconds(1), + ); + assert!(expired.bearer().await.is_err()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn unchanged_legacy_hermes_client_uses_https_standard_files_logs_and_proposals() { + let f = fixture().await; + let python = std::env::var("SRE_TEST_PYTHON").unwrap_or_else(|_| "python3".into()); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../runtimes/hermes/src/kars_runtime_hermes/plugin/sre_kube.py"); + let port = reqwest::Url::parse(&f.url).unwrap().port().unwrap(); + let result=tokio::process::Command::new(python) + .env("KUBERNETES_SERVICE_HOST","127.0.0.1").env("KUBERNETES_SERVICE_PORT",port.to_string()) + .env("NO_PROXY","127.0.0.1,localhost").env("no_proxy","127.0.0.1,localhost") + .env("AGENT_FILES",f.directory.path()).env("LEGACY_SOURCE",source) + .arg("-c").arg(r#" +import importlib.util, os, pathlib, sys, types +package = types.ModuleType("legacy"); package.__path__ = [] +sys.modules["legacy"] = package +spec = importlib.util.spec_from_file_location("legacy.sre_kube", os.environ["LEGACY_SOURCE"]) +module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) +sys.modules["legacy.sre_kube"] = module +module._SA_DIR = pathlib.Path(os.environ["AGENT_FILES"]) +client = module.KubeClient() +secret = client.get("/api/v1/namespaces/kars-demo/secrets/router-services-admin") +assert secret["data"] == {"control-token": ""} +assert "annotations" not in secret["metadata"] +assert client._ensure_client().get("/api/v1/namespaces/kars-demo/pods/app/log").text == "legitimate pod log\n" +proposal = client.post("/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions", json={ + "apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction","metadata":{"name":"legacy"}, + "spec":{"action":{"type":"RolloutRestart","params":{}}}}) +assert proposal["spec"]["approval"] == {"state":"Pending"} +source = pathlib.Path(os.environ["LEGACY_SOURCE"]).with_name("sre.py") +spec = importlib.util.spec_from_file_location("legacy.sre", source) +sre = importlib.util.module_from_spec(spec); spec.loader.exec_module(sre) +# Execute the unchanged plugin's real builder, including generateName and its +# diagnostic labels, rather than merely posting a hand-written minimal object. +sre._create_karssreaction_cr(action={"type":"RolloutRestart","namespace":"kars-demo","name":"app"}, + diagnosis="legitimate diagnosis", rationale="restart", ttl_minutes=5) +module.client().close() +client.close() +"#).output().await.unwrap(); + assert!( + result.status.success(), + "legacy HTTPS client failed: {}", + String::from_utf8_lossy(&result.stderr) + ); +} diff --git a/shared/sre_privacy.rs b/shared/sre_privacy.rs new file mode 100644 index 000000000..dec473751 --- /dev/null +++ b/shared/sre_privacy.rs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Shared wire contract for controller issuance and private-router authorization. +//! Included by both binaries without another dependency or feature variant. + +use serde_json::{Value, json}; + +pub const REVISION: &str = "kars.azure.com/sre-privacy/v2"; + +pub fn secret_access_reviews(namespace: &str) -> Vec { + let mut reviews = Vec::new(); + for scope in [Some(namespace), None] { + for verb in ["get", "list", "watch"] { + for name in [ + None, + Some("router-services-admin"), + Some("sre-api-router-identity"), + ] { + let mut attributes = json!({"group":"","resource":"secrets","verb":verb}); + if let Some(scope) = scope { + attributes["namespace"] = scope.into(); + } + if let Some(name) = name { + attributes["name"] = name.into(); + } + reviews.push(json!({ + "apiVersion":"authorization.k8s.io/v1","kind":"SubjectAccessReview", + "spec":{"user":"system:serviceaccount:kars-sre:sandbox", + "groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:kars-sre"], + "resourceAttributes":attributes}, + })); + } + } + } + reviews +} + +pub fn require_denial(response: &Value) -> Result<(), &'static str> { + if response["status"]["allowed"].as_bool() != Some(false) + || response["status"] + .get("evaluationError") + .is_some_and(|error| !error.is_null() && error.as_str() != Some("")) + { + return Err("Legacy SRE Secret get/list/watch authorization is allowed or indeterminate"); + } + Ok(()) +} + +/// Only metadata is requested. Even an Opaque Secret carrying this reserved +/// annotation is quarantined: no type mutation or stale SA-UID alias is adopted. +pub fn reject_legacy_aliases(list: &Value, account_uids: &[&str]) -> Result<(), &'static str> { + let items = list["items"] + .as_array() + .ok_or("SRE credential metadata inventory is invalid")?; + if list["metadata"] + .get("continue") + .is_some_and(|token| !token.is_null() && token.as_str() != Some("")) + { + return Err("SRE credential metadata inventory is incomplete"); + } + for item in items { + let metadata = &item["metadata"]; + if ["name", "uid", "resourceVersion"] + .iter() + .any(|key| metadata[*key].as_str().is_none_or(str::is_empty)) + { + return Err("SRE credential metadata omitted its exact identity"); + } + let annotations = &metadata["annotations"]; + if annotations["kubernetes.io/service-account.name"] == "sre-api-router" + || annotations["kubernetes.io/service-account.uid"] + .as_str() + .is_some_and(|uid| !uid.is_empty() && account_uids.contains(&uid)) + { + return Err( + "Unsafe legacy SRE token Secret alias exists; operator review required, no Secret adopted or deleted", + ); + } + } + Ok(()) +} diff --git a/tests/e2e/namespace-ownership.sh b/tests/e2e/namespace-ownership.sh index 3972742d4..31f475c2f 100644 --- a/tests/e2e/namespace-ownership.sh +++ b/tests/e2e/namespace-ownership.sh @@ -2,60 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# Sourced only by the existing disposable Kind harness. +# Retains the original SRE source/namespace lifecycle gate, now using the +# required safe CLI stage/review/enroll/migrate/install/retire/uninstall flow. +# The fresh source is created after legacy retirement and receives new UIDs; +# retained registration history is updated with exact reviewed UID/RV CAS. test_sre_namespace_ownership() { - local context="kind-kars-e2e" system_uid sandbox_uid namespace_uid claimed_uid backlink source_ns - local k=(kubectl --context "$context") - system_uid=$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}') || { - fail "Cannot read the disposable Kind namespace"; return 1; - } - if ! helm upgrade kars "$ROOT_DIR/deploy/helm/kars" \ - --kube-context "$context" --namespace kars-system --reuse-values \ - --set sre.enabled=true --set-string runtimes.hermes.image=kars-sandbox-e2e:dev \ - --wait --timeout 3m; then - fail "Fresh SRE enable failed"; return 1; - fi - local deadline=$(($(date +%s) + 120)) - local materialized=0 - while [ "$(date +%s)" -lt "$deadline" ]; do - if "${k[@]}" get deployment sre -n kars-sre >/dev/null 2>&1 \ - && "${k[@]}" get serviceaccount sre-writer -n kars-sre >/dev/null 2>&1; then - materialized=1 - break - fi - sleep 2 - done - if [ "$materialized" -ne 1 ]; then - "${k[@]}" get karssandbox sre -n kars-system -o yaml || true - fail "SRE did not materialize its deployment and writer account after namespace claiming" - return 1 - fi - sandbox_uid=$("${k[@]}" get karssandbox sre -n kars-system -o jsonpath='{.metadata.uid}') || return 1 - namespace_uid=$("${k[@]}" get namespace kars-sre -o jsonpath='{.metadata.uid}') || return 1 - claimed_uid=$("${k[@]}" get namespace kars-sre -o go-template='{{index .metadata.annotations "kars.azure.com/sandbox-uid"}}') || return 1 - source_ns=$("${k[@]}" get namespace kars-sre -o go-template='{{index .metadata.annotations "kars.azure.com/sandbox-namespace"}}') || return 1 - backlink=$("${k[@]}" get karssandbox sre -n kars-system -o go-template='{{index .metadata.annotations "kars.azure.com/namespace-uid"}}') || return 1 - if [ -z "$sandbox_uid" ] || [ -z "$namespace_uid" ] \ - || [ "$claimed_uid" != "$sandbox_uid" ] || [ "$backlink" != "$namespace_uid" ] \ - || [ "$source_ns" != "kars-system" ]; then - fail "Fresh SRE runtime lacks exact two-way namespace ownership"; return 1; - fi - if [ "$("${k[@]}" get serviceaccount sre-writer -n kars-sre -o jsonpath='{.automountServiceAccountToken}')" != "false" ]; then - fail "SRE writer account does not disable token automount"; return 1; - fi - pass "Fresh SRE install materializes a UID-bound namespace, deployment and non-automounting writer" - - if ! helm upgrade kars "$ROOT_DIR/deploy/helm/kars" \ - --kube-context "$context" --namespace kars-system --reuse-values \ - --set sre.enabled=false --wait --timeout 3m; then - fail "SRE disable failed"; return 1; - fi - if ! "${k[@]}" wait --for=delete karssandbox/sre -n kars-system --timeout=120s \ - || ! "${k[@]}" wait --for=delete namespace/kars-sre --timeout=120s; then - fail "SRE namespace cleanup did not complete"; return 1; - fi - if [ "$("${k[@]}" get namespace kars-system -o jsonpath='{.metadata.uid}')" != "$system_uid" ]; then - fail "SRE removal changed the core namespace identity"; return 1; - fi - pass "SRE removal completes guarded cleanup and preserves the core namespace" + sre_authority_phase fresh } diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 9e9c58cce..81e99b304 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -28,6 +28,8 @@ CLUSTER_NAME="kars-e2e" RUNTIME="${KARS_E2E_RUNTIME:-openclaw}" PASS=0 FAIL=0 +SRE_LEGACY_PREPARED=0 +E2E_KUBECONFIG="$ROOT_DIR/.e2e-kind-kubeconfig" # ─── Colors ─────────────────────────────────────────────────────────────────── RED='\033[0;31m' @@ -44,12 +46,18 @@ warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } setup_cluster() { info "Creating Kind cluster: $CLUSTER_NAME" - if kind get clusters 2>/dev/null | grep -q "$CLUSTER_NAME"; then + if kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then info "Cluster already exists, reusing" + kind get kubeconfig --name "$CLUSTER_NAME" > "$E2E_KUBECONFIG" + chmod 600 "$E2E_KUBECONFIG" + export KUBECONFIG="$E2E_KUBECONFIG" return fi kind create cluster --name "$CLUSTER_NAME" --config "$SCRIPT_DIR/kind-config.yaml" + kind get kubeconfig --name "$CLUSTER_NAME" > "$E2E_KUBECONFIG" + chmod 600 "$E2E_KUBECONFIG" + export KUBECONFIG="$E2E_KUBECONFIG" info "Cluster created" } @@ -171,6 +179,13 @@ install_crds() { --set-string "controller.extraEnv[0].value=false" ) fi + if [ "$SRE_LEGACY_PREPARED" = "1" ]; then + # Legacy fixtures predate the new admission policies. Explicit CLI + # authority staging already installed those APIs with controller=0; + # start the qualified controller while retaining the reviewed shapes. + extra_set_args+=(--set sre.enabled=true --set sre.authorityStage=true + --set-string runtimes.hermes.image=kars-sandbox-e2e:dev) + fi if ! helm upgrade --install kars "$ROOT_DIR/deploy/helm/kars" \ --namespace kars-system \ --create-namespace \ @@ -193,8 +208,10 @@ install_crds() { } teardown() { + sre_authority_cleanup || true info "Tearing down Kind cluster" kind delete cluster --name "$CLUSTER_NAME" 2>/dev/null || true + rm -f "$E2E_KUBECONFIG" } # ─── Tests ──────────────────────────────────────────────────────────────────── @@ -3015,21 +3032,29 @@ EOF # ─── Main ───────────────────────────────────────────────────────────────────── +source "$SCRIPT_DIR/sre-authority.sh" source "$SCRIPT_DIR/namespace-ownership.sh" source "$SCRIPT_DIR/credential-sources.sh" main() { + umask 077 echo "" echo "═══════════════════════════════════════════════════════" echo " kars E2E Test Suite (runtime: $RUNTIME)" echo "═══════════════════════════════════════════════════════" echo "" + PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$SCRIPT_DIR" \ + python3 -m unittest discover -s "$SCRIPT_DIR/sre_authority" -p '*_test.py' trap teardown EXIT setup_cluster build_images + prepare_sre_authority_legacy install_crds + # Finish real legacy retirement before unrelated tests can create private + # namespaces/credentials. Failure is fatal, not a skipped/false-positive gate. + test_sre_authority_migration echo "" info "Running tests..." diff --git a/tests/e2e/sre-authority.py b/tests/e2e/sre-authority.py new file mode 100644 index 000000000..0b0edead5 --- /dev/null +++ b/tests/e2e/sre-authority.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import argparse +from pathlib import Path + + +def cleanup(): + work = Path(__file__).resolve().parents[2] / ".e2e-sre-authority" + # Remove only explicitly owned credential files, never broad namespaces, + # arbitrary directories, or another process's kubeconfig. + for name in ("admin.json", "registrar.json", "tenant.json", "normal.json", + "old-agent.json", "watch-only.json", "unrelated.json", "opaque.json", "admin-key.pem", + "admin-cert.pem", "api-ca.pem", "agent/token", "agent/ca.crt", "agent/namespace"): + (work / name).unlink(missing_ok=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("phase", choices=("prepare", "legacy", "fresh", "cleanup")) + options = parser.parse_args() + if options.phase == "cleanup": + cleanup() + return 0 + from sre_authority.common import Harness, SYSTEM + from sre_authority.fixtures import CONTROL, CONTROL_NS, prepare_legacy + from sre_authority.migration import fresh_reenrollment, legacy_migration, retire_and_uninstall + from sre_authority.proxy import proxy_acceptance + harness = None + try: + harness = Harness(options.phase) + if options.phase == "prepare": + prepare_legacy(harness) + elif options.phase == "legacy": + legacy_migration(harness) + proxy_acceptance(harness) + retire_and_uninstall(harness) + source = harness.get("karssandbox", CONTROL, SYSTEM) + if source: + harness.api("DELETE", f"/apis/kars.azure.com/v1alpha1/namespaces/{SYSTEM}/karssandboxes/{CONTROL}", + body={"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": { + "uid": harness.state["control_source_uid"], + "resourceVersion": source["metadata"]["resourceVersion"]}}, status=(200, 202)) + harness.poll("owned control fixture cleanup", lambda: harness.get("namespace", CONTROL_NS) is None, seconds=120) + else: + fresh_reenrollment(harness) + proxy_acceptance(harness) + retire_and_uninstall(harness) + harness.save() + return 0 + except Exception as error: + # Report only the exception class and controlled assertions. API bodies, + # command output, JWTs and TLS private keys never become failure logs. + print(f"SRE-FAIL {options.phase}: {str(error) if isinstance(error, AssertionError) else type(error).__name__}", flush=True) + if harness: + harness.diagnostics() + return 1 + finally: + if harness: + harness.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/sre-authority.sh b/tests/e2e/sre-authority.sh new file mode 100644 index 000000000..fd4376c87 --- /dev/null +++ b/tests/e2e/sre-authority.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Called only by the explicit disposable Kind harness. No live/default context. +sre_authority_phase() { + local phase="$1" output result=0 line + info "SRE authority acceptance: ${phase}" + output=$(python3 "$SCRIPT_DIR/sre-authority.py" "$phase") || result=$? + while IFS= read -r line; do + case "$line" in + "SRE-PASS "*) pass "${line#SRE-PASS }" ;; + "SRE-FAIL "*) fail "${line#SRE-FAIL }" ;; + *) [ -z "$line" ] || printf '%s\n' "$line" ;; + esac + done <<< "$output" + return "$result" +} + +prepare_sre_authority_legacy() { + sre_authority_phase prepare || return + SRE_LEGACY_PREPARED=1 +} + +test_sre_authority_migration() { + sre_authority_phase legacy +} + +sre_authority_cleanup() { + python3 "$SCRIPT_DIR/sre-authority.py" cleanup +} diff --git a/tests/e2e/sre_authority/__init__.py b/tests/e2e/sre_authority/__init__.py new file mode 100644 index 000000000..732a1699c --- /dev/null +++ b/tests/e2e/sre_authority/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Hosted disposable-Kind acceptance, not an agent/model execution harness.""" diff --git a/tests/e2e/sre_authority/admission.py b/tests/e2e/sre_authority/admission.py new file mode 100644 index 000000000..9cd2e9cbd --- /dev/null +++ b/tests/e2e/sre_authority/admission.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json + +from .common import POLICIES, PRIVATE, REGISTRATION, RUNTIME, STANDIN, TENANT, assert_denial, require + + +def policies_ready(h): + def checked(): + for suffix in POLICIES: + name = f"kars-sre-{suffix}" + policy = h.get("validatingadmissionpolicy", name) + binding = h.get("validatingadmissionpolicybinding", name) + if not policy or not binding: + return False + status = policy.get("status", {}) + if status.get("observedGeneration") != policy["metadata"]["generation"] or "typeChecking" not in status: + return False + require(not status["typeChecking"].get("expressionWarnings"), f"CEL type warnings in {name}") + require(policy["spec"].get("failurePolicy") == "Fail", f"{name} does not fail closed") + require(binding["spec"]["policyName"] == name and "Deny" in binding["spec"]["validationActions"], + f"{name} lacks a matching enforcing binding") + return True + h.poll("all observed/enforcing SRE CEL policies", checked, seconds=90) + h.passed(f"All {len(POLICIES)} real API admission policies are observed, CEL-type-checked without warnings, and Deny-bound") + + +def pod_spec(private=True): + spec = {"serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "securityContext": {"runAsNonRoot": True, "runAsUser": 1000, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "probe", "image": STANDIN, "command": ["/bin/sh", "-c", "sleep infinity"], + "securityContext": {"allowPrivilegeEscalation": False, "capabilities": {"drop": ["ALL"]}}}]} + if private: + spec["volumes"] = [{"name": "private", "secret": {"secretName": PRIVATE}}] + spec["containers"][0]["volumeMounts"] = [{"name": "private", "mountPath": "/private", "readOnly": True}] + return spec + + +def admission_cases(h, enrollment): + registration = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, "spec": enrollment} + # Normal SA has no cluster enrollment permission. Tenant probe has CREATE + # but deliberately lacks USE, so the second check exercises actual CEL. + assert_denial(h.api("POST", REGISTRATION.rsplit("/", 1)[0] + "?dryRun=All", + body=registration, user="normal"), "ordinary registration RBAC") + assert_denial(h.api("POST", REGISTRATION.rsplit("/", 1)[0] + "?dryRun=All", + body=registration, user="tenant"), "registration CEL", "kars-sre-registration-authority") + review = h.api("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", user="normal", body={ + "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", + "spec": {"resourceAttributes": {"group": "kars.azure.com", "resource": "karssreregistrations", + "name": "canonical", "verb": "use"}}}, status=201).json() + require(review["status"]["allowed"] is False, "Ordinary SA unexpectedly has registrar use") + h.passed("Real RBAC and CEL independently deny ordinary/tenant registration and registrar use") + + source = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": "sre", "namespace": TENANT}, + "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1"}}, + "sandbox": {"isolation": "standard"}}} + assert_denial(h.api("POST", f"/apis/kars.azure.com/v1alpha1/namespaces/{TENANT}/karssandboxes?dryRun=All", + body=source, user="tenant"), "reserved source", "kars-sre-source-authority") + source["metadata"]["name"] = "not-canonical" + source["metadata"]["labels"] = {"kars.azure.com/role": "sre"} + assert_denial(h.api("POST", f"/apis/kars.azure.com/v1alpha1/namespaces/{TENANT}/karssandboxes?dryRun=All", + body=source, user="tenant"), "reserved SRE role", "kars-sre-source-authority") + h.passed("Namespace authority cannot create a reserved-name or SRE-labelled source") + + sa = {"apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": "sre-api-router", "namespace": TENANT}} + assert_denial(h.api("POST", f"/api/v1/namespaces/{TENANT}/serviceaccounts?dryRun=All", + body=sa, user="tenant"), "reserved ServiceAccount", "kars-sre-private-identity") + secret = {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": PRIVATE, "namespace": RUNTIME}, + "stringData": {"kube-token": "not-a-credential"}} + assert_denial(h.api("POST", f"/api/v1/namespaces/{RUNTIME}/secrets?dryRun=All", + body=secret, user="tenant"), "reserved material", "kars-sre-private-material") + h.passed("Real admission denies reserved identities and private material despite namespaced create permission") + + # An otherwise valid ordinary Pod is admitted in server-side dry-run; + # private variants must be denied by the intended policy, not schema/RBAC. + ordinary = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "e2e-ordinary", "namespace": RUNTIME}, "spec": pod_spec(False)} + h.api("POST", f"/api/v1/namespaces/{RUNTIME}/pods?dryRun=All", body=ordinary, user="tenant", status=201) + private = {**ordinary, "metadata": {"name": "e2e-private", "namespace": RUNTIME}, "spec": pod_spec()} + assert_denial(h.api("POST", f"/api/v1/namespaces/{RUNTIME}/pods?dryRun=All", + body=private, user="tenant"), "private Pod mount", "kars-sre-private-mounts") + env_only = pod_spec(False) + env_only["containers"][0]["envFrom"] = [{"secretRef": {"name": PRIVATE}}] + assert_denial(h.api("POST", f"/api/v1/namespaces/{RUNTIME}/pods?dryRun=All", + body={**private, "spec": env_only}, user="tenant"), "private envFrom", "kars-sre-private-mounts") + h.passed("A valid ordinary Pod is accepted; private volume/envFrom laundering is denied") + + for kind, group, plural in [("Deployment", "apps/v1", "deployments"), ("ReplicaSet", "apps/v1", "replicasets"), + ("Job", "batch/v1", "jobs"), ("CronJob", "batch/v1", "cronjobs")]: + template = {"metadata": {"labels": {"app": "e2e-private-template"}}, "spec": pod_spec()} + if kind in ("Job", "CronJob"): + template["spec"]["restartPolicy"] = "Never" + spec = {"template": template} + if kind in ("Deployment", "ReplicaSet"): + spec.update({"replicas": 1, "selector": {"matchLabels": template["metadata"]["labels"]}}) + if kind == "CronJob": + spec = {"schedule": "0 * * * *", "jobTemplate": {"spec": spec}} + obj = {"apiVersion": group, "kind": kind, "metadata": {"name": "e2e-private-template", "namespace": RUNTIME}, "spec": spec} + policy = "kars-sre-private-cronjobs" if kind == "CronJob" else "kars-sre-private-workloads" + assert_denial(h.api("POST", f"/apis/{group}/namespaces/{RUNTIME}/{plural}?dryRun=All", + body=obj, user="tenant"), f"{kind} private template", policy) + h.passed(f"Real {kind} admission prevents private-material laundering through workload controllers") + + +def runtime_denials(h, pod): + response = h.api("POST", f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router/token", + body={"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest", + "spec": {"audiences": [], "expirationSeconds": 600}}, user="tenant") + assert_denial(response, "private TokenRequest", "kars-sre-private-identity") + for subresource, args in [ + ("exec", ["exec", "-n", RUNTIME, pod, "-c", "agent", "--", "/bin/true"]), + ("attach", ["attach", "-n", RUNTIME, pod, "-c", "agent"]), + ("portforward", ["port-forward", "-n", RUNTIME, f"pod/{pod}", "19446:9446"]), + ]: + result = h.k(*args, user="tenant", expected=None, timeout=25) + require(result.returncode != 0 and "kars-sre-private-connect" in result.stderr + and "forbidden" in result.stderr.lower(), f"{subresource} did not receive the intended connect-admission denial") + assert_denial(h.api("GET", f"/api/v1/namespaces/{RUNTIME}/pods/{pod}:9446/proxy/api/v1/namespaces", + user="tenant"), "Pod proxy connect", "kars-sre-private-connect") + h.passed("Tenant TokenRequests and actual exec/attach/port-forward/proxy attempts receive intended admission denials") diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py new file mode 100644 index 000000000..8ca156e72 --- /dev/null +++ b/tests/e2e/sre_authority/common.py @@ -0,0 +1,348 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import base64 +import contextlib +import hashlib +import json +import os +from pathlib import Path +import socket +import signal +import ssl +import subprocess +import time + +CONTEXT = "kind-kars-e2e" +SYSTEM = "kars-system" +RUNTIME = "kars-sre" +OPERATORS = "e2e-sre-operators" +TENANT = "e2e-sre-tenant" +REGISTRATION = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical" +PRIVATE = "sre-api-router-identity" +AGENT = "sre-api-agent" +CLAIM_VERSION = "kars.azure.com/namespace-claim-version" +SOURCE_NS = "kars.azure.com/sandbox-namespace" +SOURCE_NAME = "kars.azure.com/sandbox-name" +SOURCE_UID = "kars.azure.com/sandbox-uid" +NAMESPACE_UID = "kars.azure.com/namespace-uid" +EPOCH = "kars.azure.com/sre-privacy-epoch" +OWNER = "kars.azure.com/sre-registration-uid" +PRIVACY_REVISION = "kars.azure.com/sre-privacy/v2" +FIELD_MANAGER = "kars-controller/karssandbox" +STANDIN = "kars-sandbox-e2e:dev" +POLICIES = ( + "source-authority", "registration-authority", "private-identity", + "binding-authority", "private-material", "source-retirement", + "pending-proposals", "consumer-authority", "private-mounts", + "private-workloads", "private-cronjobs", "private-connect", "role-authority", "no-legacy-tokens", +) + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def printed_object(output): + start = output.find("{") + require(start >= 0, "CLI omitted its JSON object") + value, _ = json.JSONDecoder().raw_decode(output[start:]) + require(isinstance(value, dict), "CLI returned a non-object JSON value") + return value + + +def enrollment_json(output): + value = printed_object(output) + require(all(key in value for key in ("controller", "sandbox", "runtimeNamespace", "legacyBindings")), + "CLI preview returned an unexpected object") + return value + + +def assert_claim(source, namespace, core, system_uid): + require(source and namespace and core, "Source/namespace/core identity is missing") + src, ns = source["metadata"], namespace["metadata"] + require(src.get("uid") and ns.get("uid") and src.get("resourceVersion") and ns.get("resourceVersion"), + "Source or namespace lacks its real API identity") + require(src.get("namespace") == SYSTEM and src.get("name") == "sre" and ns.get("name") == RUNTIME, + "Claim points outside the canonical source/runtime namespace") + require(not src.get("deletionTimestamp") and not ns.get("deletionTimestamp") and not ns.get("ownerReferences"), + "Claim is terminating or has a foreign owner") + annotations = ns.get("annotations", {}) + require(all(annotations.get(key) == value for key, value in { + CLAIM_VERSION: "v1", SOURCE_NS: SYSTEM, SOURCE_NAME: "sre", SOURCE_UID: src["uid"], + }.items()) and "kars.azure.com/namespace-prestage" not in annotations, + "Actual namespace lacks the complete current-source claim") + require(src.get("annotations", {}).get(NAMESPACE_UID) == ns["uid"], + "Actual source lacks the exact runtime-namespace UID backlink") + require(core["metadata"].get("uid") == system_uid, "Core namespace identity changed") + + +def review_args(spec, registration=None): + args = ["--sandbox-uid", spec["sandbox"]["uid"], + "--namespace-uid", spec["runtimeNamespace"]["uid"]] + for binding in spec["legacyBindings"]: + args += ["--binding", f'{binding["kind"]}/{binding.get("namespace", "")}/{binding["name"]}=' + f'{binding["uid"]}@{binding["resourceVersion"]}'] + if spec.get("legacyConsumer"): + consumer = spec["legacyConsumer"] + args += ["--consumer", f'{consumer["uid"]}@{consumer["resourceVersion"]}'] + if registration: + args += ["--registration-uid", registration["metadata"]["uid"], + "--resource-version", registration["metadata"]["resourceVersion"]] + return args + + +def assert_denial(response, label, policy=None): + # A missing CRD/resource, transport failure or malformed fixture is NOT proof. + require(response.status_code == 403, f"{label}: expected Forbidden, got HTTP {response.status_code}") + body = response.json() + require(body.get("kind") == "Status" and body.get("reason") == "Forbidden", + f"{label}: response was not a Kubernetes authorization/admission denial") + if policy: + require(policy in body.get("message", ""), f"{label}: wrong admission policy rejected the fixture") + + +class Harness: + def __init__(self, phase): + import httpx + os.umask(0o077) + self.httpx = httpx + self.root = Path(__file__).resolve().parents[3] + self.work = self.root / ".e2e-sre-authority" + self.work.mkdir(mode=0o700, exist_ok=True) + os.chmod(self.work, 0o700) + self.deadline = time.monotonic() + 650 + self.phase = phase + self.clients = {} + self.processes = [] + self.state_path = self.work / "state.json" + self.state = json.loads(self.state_path.read_text()) if self.state_path.exists() else {} + config_path = self.work / "admin.json" + if not config_path.exists(): + raw = self.run(["kubectl", "--context", CONTEXT, "--request-timeout=15s", + "config", "view", "--raw", "--minify", "-o", "json"], timeout=20) + config = json.loads(raw) + require(config["contexts"][0]["name"] == CONTEXT, "Refusing a non-Kind Kubernetes context") + config["current-context"] = CONTEXT + cluster = config["clusters"][0]["cluster"] + from urllib.parse import urlsplit + require(urlsplit(cluster["server"]).hostname in ("127.0.0.1", "::1", "localhost"), + "Refusing a non-loopback Kind API server") + self.write("admin.json", json.dumps(config)) + self.config = json.loads(config_path.read_text()) + cluster = self.config["clusters"][0]["cluster"] + self.server = cluster["server"] + self.write("api-ca.pem", base64.b64decode(cluster["certificate-authority-data"])) + admin = self.config["users"][0]["user"] + require("client-certificate-data" in admin and "client-key-data" in admin, + "Kind admin configuration must use its client certificate") + self.write("admin-cert.pem", base64.b64decode(admin["client-certificate-data"])) + self.write("admin-key.pem", base64.b64decode(admin["client-key-data"])) + + def write(self, name, value): + path = self.work / name + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "wb") as output: + output.write(value if isinstance(value, bytes) else value.encode()) + os.chmod(path, 0o600) + return path + + def save(self): + self.write("state.json", json.dumps(self.state, indent=2)) + + def run(self, args, *, data=None, user="admin", timeout=35, expected=0): + env = {**os.environ, "KARS_KUBE_CONTEXT": CONTEXT, "NO_COLOR": "1", "FORCE_COLOR": "0"} + config = self.work / f"{user}.json" + if config.exists(): + env["KUBECONFIG"] = str(config) + else: + require(user == "admin", "Probe principal has no isolated kubeconfig") + remaining = self.deadline - time.monotonic() + require(remaining > 0, "SRE acceptance phase exceeded its bounded deadline") + process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, cwd=self.root, + env=env, start_new_session=True) + try: + stdout, stderr = process.communicate(input=data, timeout=min(timeout, remaining)) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGTERM) + try: + process.communicate(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=5) + raise AssertionError(f"Command {Path(args[0]).name} exceeded its bounded timeout") from None + result = subprocess.CompletedProcess(args, process.returncode, stdout, stderr) + if expected is not None: + # Never echo command output or argv: token/Secret reads are captured. + require(result.returncode == expected, f"Command {Path(args[0]).name} failed during {self.phase}") + return result if expected is None else result.stdout + + def k(self, *args, data=None, user="admin", timeout=35, expected=0): + request_timeout = max(1, int(min(timeout - 2, self.deadline - time.monotonic()))) + return self.run(["kubectl", "--context", CONTEXT, f"--request-timeout={request_timeout}s", *args], + data=data, user=user, timeout=timeout, expected=expected) + + def cli(self, *args, user="admin", expected=0, timeout=240): + return self.run(["node", str(self.root / "cli/dist/index.js"), "sre", *args, + "--context", CONTEXT, "--namespace", SYSTEM, "--release", "kars"], + user=user, expected=expected, timeout=timeout) + + def client(self, user="admin"): + if user not in self.clients: + context = ssl.create_default_context(cafile=str(self.work / "api-ca.pem")) + headers = {"Accept": "application/json"} + if user == "admin": + context.load_cert_chain(str(self.work / "admin-cert.pem"), str(self.work / "admin-key.pem")) + else: + # Deliberately separate SSLContext: wrong-token probes MUST NOT + # retain an admin certificate, exec plugin or auth-provider. + config = json.loads((self.work / f"{user}.json").read_text()) + account = config["users"][0]["user"] + require(set(account) == {"token"}, "Non-admin probe retained privileged kubeconfig auth") + headers["Authorization"] = f'Bearer {account["token"]}' + self.clients[user] = self.httpx.Client(base_url=self.server, verify=context, + headers=headers, timeout=15, trust_env=False) + return self.clients[user] + + def api(self, method, path, *, body=None, user="admin", status=None): + headers = {"Content-Type": "application/merge-patch+json"} if method == "PATCH" else None + response = self.client(user).request(method, path, json=body, headers=headers) + if status is not None: + require(response.status_code in (status if isinstance(status, tuple) else (status,)), + f"{method} {path.split('?')[0]}: HTTP {response.status_code}, expected {status}") + return response + + def get(self, kind, name, namespace=None): + args = ["get", kind, name, "--ignore-not-found", "-o", "json"] + if namespace: + args += ["-n", namespace] + raw = self.k(*args) + return json.loads(raw) if raw.strip() else None + + def create(self, obj, manager=None): + args = ["create", "-f", "-", "-o", "json"] + if manager: + args += ["--field-manager", manager] + return json.loads(self.k(*args, data=json.dumps(obj))) + + def token_identity(self, name, namespace, account): + token = self.k("create", "token", account, "-n", namespace, "--duration=1h").strip() + require(token, "TokenRequest returned no credential") + return self.token_config(name, token) + + def token_config(self, name, token): + cluster = self.config["clusters"][0] + self.write(f"{name}.json", json.dumps({ + "apiVersion": "v1", "kind": "Config", "current-context": CONTEXT, + "clusters": [cluster], "users": [{"name": name, "user": {"token": token}}], + "contexts": [{"name": CONTEXT, "context": {"cluster": cluster["name"], "user": name}}], + })) + old = self.clients.pop(name, None) + if old: + old.close() + + def poll(self, label, predicate, seconds=150, interval=1): + end = min(self.deadline, time.monotonic() + seconds) + while time.monotonic() < end: + value = predicate() + if value: + return value + time.sleep(interval) + raise AssertionError(f"{label} did not converge before its deadline") + + def wait_ready(self): + def ready(): + reg = self.get("karssreregistrations.kars.azure.com", "canonical") + if not reg: + return False + status = reg.get("status", {}) + require(not (status.get("phase") == "Blocked" and status.get("observedGeneration") == reg["metadata"]["generation"]), + "SRE authority reported Blocked; see sanitized status diagnostics") + return reg if (status.get("phase") == "Ready" + and status.get("observedGeneration") == reg["metadata"]["generation"] + and status.get("privacyRevision") == PRIVACY_REVISION + and status.get("legacySecretAccessDenied") is True) else False + return self.poll("SRE Ready", ready, seconds=240) + + def namespace_delete(self, name, uid): + obj = self.get("namespace", name) + require(obj and obj["metadata"]["uid"] == uid, "Fixture namespace changed before cleanup") + self.api("DELETE", f"/api/v1/namespaces/{name}", body={ + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": obj["metadata"]["resourceVersion"]}, + }, status=(200, 202)) + self.poll(f"namespace {name} cleanup", lambda: self.get("namespace", name) is None, seconds=120) + + @contextlib.contextmanager + def port_forward(self, pod): + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + log = self.work / "port-forward.log" + with log.open("w") as output: + process = subprocess.Popen(["kubectl", "--context", CONTEXT, "--request-timeout=20s", + "-n", RUNTIME, "port-forward", f"pod/{pod}", f"{port}:9446", "--address=127.0.0.1"], + cwd=self.root, env={**os.environ, "KUBECONFIG": str(self.work / "registrar.json")}, + stdout=output, stderr=output) + self.processes.append(process) + try: + self.poll("private TLS port-forward", lambda: process.poll() is None + and log.exists() and "Forwarding from" in log.read_text(), seconds=25) + yield port + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + self.processes.remove(process) + + def passed(self, message): + print(f"SRE-PASS {message}", flush=True) + + def diagnostics(self): + self.deadline = max(self.deadline, time.monotonic() + 50) + # Status and identities only; never dump Secret bodies or whole Pods. + for kind, name, namespace in [("karssreregistrations.kars.azure.com", "canonical", None), + ("karssandbox", "sre", SYSTEM), ("deployment", "sre", RUNTIME)]: + try: + obj = self.get(kind, name, namespace) + if obj: + status = obj.get("status", {}) + print("SRE-DIAG", json.dumps({"kind": kind, "name": name, + "uid": obj["metadata"].get("uid"), "phase": status.get("phase"), + "observedGeneration": status.get("observedGeneration"), + "generation": obj["metadata"].get("generation"), + "availableReplicas": status.get("availableReplicas"), + "conditions": [{"type": condition.get("type"), "status": condition.get("status"), + "reason": condition.get("reason")} + for condition in status.get("conditions", [])]}), flush=True) + except Exception: + print(f"SRE-DIAG {kind}/{name} unavailable", flush=True) + try: + pods = json.loads(self.k("get", "pods", "-n", RUNTIME, "-o", "json", timeout=10))["items"] + for pod in pods: + status = pod.get("status", {}) + containers = status.get("initContainerStatuses", []) + status.get("containerStatuses", []) + print("SRE-DIAG", json.dumps({"kind": "Pod", "name": pod["metadata"]["name"], + "uid": pod["metadata"]["uid"], "phase": status.get("phase"), + "containers": [{"name": container["name"], "ready": container.get("ready"), + "state": {kind: {key: value.get(key) for key in ("reason", "exitCode") if key in value} + for kind, value in container.get("state", {}).items()}} + for container in containers]}), flush=True) + except Exception: + print("SRE-DIAG runtime Pod status unavailable", flush=True) + + def close(self): + for process in self.processes: + process.terminate() + for client in self.clients.values(): + client.close() + + +def fingerprint(secret, key="control-token"): + return hashlib.sha256(base64.b64decode(secret["data"][key])).hexdigest() diff --git a/tests/e2e/sre_authority/credential_paths.py b/tests/e2e/sre_authority/credential_paths.py new file mode 100644 index 000000000..d30155f39 --- /dev/null +++ b/tests/e2e/sre_authority/credential_paths.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import base64 +from concurrent.futures import ThreadPoolExecutor +import copy +import json +import threading +import time + +from .common import AGENT, PRIVATE, REGISTRATION, RUNTIME, SYSTEM, assert_denial, require + +WATCH_BINDING = "e2e-sre-legacy-watch-only" +TOKEN_ALIAS = "e2e-prestaged-router-token" +TOKEN_TYPE = "kubernetes.io/service-account-token" +SA_NAME = "kubernetes.io/service-account.name" +WATCH_MARKER = "kind-dummy-secret-watch-proof" + + +def seed_privacy_gaps(h): + require(h.get("serviceaccount", "sre-api-router", RUNTIME) is None, + "Legacy token alias must precede private ServiceAccount creation") + alias = h.create({"apiVersion": "v1", "kind": "Secret", "type": TOKEN_TYPE, + "metadata": {"name": TOKEN_ALIAS, "namespace": RUNTIME, "annotations": {SA_NAME: "sre-api-router"}}}) + require(not alias.get("data"), "Prestaged token alias unexpectedly contains credentials") + h.state["prestaged_alias"] = alias + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": WATCH_BINDING}, + "rules": [{"apiGroups": [""], "resources": ["secrets"], "verbs": ["watch"]}]}) + binding = h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", + "metadata": {"name": WATCH_BINDING}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": WATCH_BINDING}, + "subjects": [{"kind": "Group", "name": "system:serviceaccounts:kars-sre", + "apiGroup": "rbac.authorization.k8s.io"}]}) + h.state["watch_binding"] = binding + h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "e2e-watch-only", "namespace": RUNTIME}}) + h.token_identity("watch-only", RUNTIME, "e2e-watch-only") + h.passed("Watch-only group grant and arbitrary token alias predate admission and private ServiceAccount creation") + + +def delete_owned(h, path, obj): + h.api("DELETE", path, body={"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": obj["metadata"]["uid"], + "resourceVersion": obj["metadata"]["resourceVersion"]}}, status=(200, 202)) + + +def token_secret_denials(h, before_enrollment=False): + policy = "kars-sre-no-legacy-tokens" + path = f"/api/v1/namespaces/{RUNTIME}/secrets" + prefix = f"e2e-token-{h.phase}-{'pre' if before_enrollment else 'live'}" + unsafe = {"apiVersion": "v1", "kind": "Secret", "type": TOKEN_TYPE, + "metadata": {"name": f"{prefix}-create", "namespace": RUNTIME, + "annotations": {SA_NAME: "sre-api-router"}}} + # Real CREATE/PATCH, not a fixed private name or TokenRequest. A broken + # policy fails the disposable cluster immediately, without reading tokens. + for user in ("tenant", "registrar"): + assert_denial(h.api("POST", path, body=unsafe, user=user), + f"{user} arbitrary token Secret CREATE", policy) + fixtures = [] + try: + variants = [ + ("type-and-annotation", "Opaque", {}, {"type": TOKEN_TYPE, + "metadata": {"annotations": {SA_NAME: "sre-api-router"}}}), + ("annotation", TOKEN_TYPE, {SA_NAME: "e2e-never-created-account"}, + {"metadata": {"annotations": {SA_NAME: "sre-api-router"}}}), + ] + if before_enrollment: + # Test a type-only update before enrollment: an Opaque alias is + # itself quarantined by the controller inventory, so it must not + # be introduced into an already-Ready runtime even for this test. + variants.append(("type", "Opaque", {SA_NAME: "sre-api-router"}, {"type": TOKEN_TYPE})) + for suffix, kind, annotations, patch in variants: + obj = h.api("POST", path, body={"apiVersion": "v1", "kind": "Secret", "type": kind, + "metadata": {"name": f"{prefix}-{suffix}", "namespace": RUNTIME, + "annotations": annotations}}, user="tenant", status=201).json() + fixtures.append(obj) + assert_denial(h.api("PATCH", path + "/" + obj["metadata"]["name"], body={ + **patch, "metadata": {**patch.get("metadata", {}), + "uid": obj["metadata"]["uid"], "resourceVersion": obj["metadata"]["resourceVersion"]}}, + user="tenant"), f"token Secret {suffix} PATCH", policy) + current = h.get("secret", obj["metadata"]["name"], RUNTIME) + require(current["metadata"]["resourceVersion"] == obj["metadata"]["resourceVersion"] + and not current.get("data"), "Denied token Secret update mutated or populated its fixture") + if before_enrollment: + for patch in ({"type": "Opaque"}, {"metadata": {"annotations": {SA_NAME: "e2e-other"}}}): + assert_denial(h.api("PATCH", path + "/" + TOKEN_ALIAS, body=patch, user="tenant"), + "prestaged oldObject type/annotation escape", policy) + finally: + for obj in fixtures: + delete_owned(h, path + "/" + obj["metadata"]["name"], obj) + h.passed("Tenant and registrar arbitrary token Secret CREATE, plus type/annotation PATCH escapes, receive exact admission denials") + + +def fixture_review(h): + # Only the known pre-guard fixture is reviewed here. CLI preview must + # refuse while the broad watch grant exists; that refusal is tested too. + spec = copy.deepcopy(h.state["legacy_review_before_guards"]) + for kind, name, namespace, uid in [ + ("namespace", SYSTEM, None, spec["controller"]["namespace"]["uid"]), + ("deployment", "kars-controller", SYSTEM, spec["controller"]["deployment"]["uid"]), + ("karssandbox", "sre", SYSTEM, spec["sandbox"]["uid"]), + ("namespace", RUNTIME, None, spec["runtimeNamespace"]["uid"]), + ]: + obj = h.get(kind, name, namespace) + require(obj and obj["metadata"]["uid"] == uid, "Pre-guard fixture identity changed during staging") + for binding in spec["legacyBindings"]: + obj = h.get(binding["kind"].lower(), binding["name"], binding.get("namespace")) + require(obj and obj["metadata"]["uid"] == binding["uid"] + and obj["roleRef"] == binding["roleRef"] and obj["subjects"] == binding["subjects"], + "Pre-guard fixture binding was replaced or its reviewed authority changed") + binding["resourceVersion"] = obj["metadata"]["resourceVersion"] + consumer = h.get("deployment", "sre", RUNTIME) + require(consumer["metadata"]["uid"] == spec["legacyConsumer"]["uid"], "Pre-guard consumer was replaced") + spec["legacyConsumer"]["resourceVersion"] = consumer["metadata"]["resourceVersion"] + return spec + + +def wait_blocked(h, detail): + def blocked(): + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + status = reg.get("status", {}) if reg else {} + return reg if (status.get("phase") == "Blocked" + and status.get("observedGeneration") == reg["metadata"]["generation"] + and detail in status.get("detail", "")) else False + return h.poll(f"controller rejection for {detail}", blocked, seconds=75) + + +def assert_unissued(h, spec, before): + for binding in spec["legacyBindings"]: + require(h.get(binding["kind"].lower(), binding["name"], binding.get("namespace")) == before[binding["name"]], + "Blocked migration changed a legacy grant") + require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, + "Blocked migration stopped the legacy consumer") + require(h.get("serviceaccount", "sre-api-router", RUNTIME) is None, + "Private ServiceAccount appeared while legacy privacy was unsafe") + require(all(h.get("secret", name, RUNTIME) is None for name in (PRIVATE, AGENT)), + "Private credential appeared while legacy privacy was unsafe") + for name in ("kars-sre-private-reader", "kars-sre-private-author", "kars-sre-private-renew"): + require(h.get("clusterrolebinding", name) is None, "Private grant appeared while legacy privacy was unsafe") + require(h.get("rolebinding", "sre-api-self-renew", RUNTIME) is None, + "Private token-renewal grant appeared while legacy privacy was unsafe") + + +def block_prestaged_paths(h, spec, before): + h.api("POST", REGISTRATION.rsplit("/", 1)[0], body={ + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, "spec": spec}, user="registrar", status=201) + wait_blocked(h, "Unsafe legacy SRE token Secret alias") + assert_unissued(h, spec, before) + alias = h.get("secret", TOKEN_ALIAS, RUNTIME) + require(alias == h.state["prestaged_alias"] and not alias.get("data"), + "Prestaged token Secret was deleted, adopted, populated or modified") + delete_owned(h, f"/api/v1/namespaces/{RUNTIME}/secrets/{TOKEN_ALIAS}", alias) + h.passed("Real controller quarantines the untouched prestaged token alias before private identity/grants/issuance; operator CAS cleanup only") + + wait_blocked(h, "broad group grant") + assert_unissued(h, spec, before) + binding = h.get("clusterrolebinding", WATCH_BINDING) + require(binding == h.state["watch_binding"], "Blocked watch-only group grant was changed or adopted") + h.passed("Real migration rejects a Secrets-watch-only group grant before retiring grants or issuing private material") + # Keep the registration deliberately incomplete before removing the last + # unsafe fixture, so cleanup cannot auto-start a successful migration. + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + partial = {**spec, "legacyBindings": spec["legacyBindings"][:-1]} + h.api("PATCH", REGISTRATION, body={"metadata": {"uid": reg["metadata"]["uid"], + "resourceVersion": reg["metadata"]["resourceVersion"]}, "spec": partial}, user="registrar", status=200) + delete_owned(h, f"/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/{WATCH_BINDING}", binding) + blocked = wait_blocked(h, "Unreviewed legacy SRE binding") + assert_unissued(h, spec, before) + return blocked + + +def assert_watch_result(response, observed, allowed): + if allowed: + require(response.status_code == 200 and observed, + "Watch-only baseline did not observe the newly created dummy Secret") + else: + # HTTP 200 with no events is still an authorized watch, never a denial. + assert_denial(response, "held old principal Secret WATCH") + require(not observed, "Denied Secret watch nevertheless exposed the dummy Secret") + + +def secret_watch(h, namespace, name, *, user="old-agent", allowed=False, cluster=False): + collection = "/api/v1/secrets" if cluster else f"/api/v1/namespaces/{namespace}/secrets" + version = h.api("GET", collection, status=200).json()["metadata"]["resourceVersion"] + client = h.client(user) + started = threading.Event() + encoded = base64.b64encode(WATCH_MARKER.encode()).decode() + + def observe(): + started.set() + end = min(h.deadline, time.monotonic() + 10) + with client.stream("GET", collection, params={"watch": "true", "resourceVersion": version, + "fieldSelector": f"metadata.name={name}", "timeoutSeconds": 5}, timeout=8) as response: + if response.status_code != 200: + response.read() + return response, False + for index, line in enumerate(response.iter_lines()): + if index > 16 or time.monotonic() > end: + break + if not line: + continue + event = json.loads(line) + obj = event.get("object", {}) + if (event.get("type") == "ADDED" and obj.get("metadata", {}).get("name") == name + and obj["metadata"].get("namespace") == namespace + and obj.get("data", {}).get("dummy") == encoded): + return response, True + return response, False + + obj = None + try: + with ThreadPoolExecutor(max_workers=1) as pool: + result = pool.submit(observe) + require(started.wait(timeout=3), "Secret watch probe did not start") + obj = h.create({"apiVersion": "v1", "kind": "Secret", + "metadata": {"name": name, "namespace": namespace}, "stringData": {"dummy": WATCH_MARKER}}) + response, observed = result.result(timeout=12) + assert_watch_result(response, observed, allowed) + finally: + if obj: + delete_owned(h, f"/api/v1/namespaces/{namespace}/secrets/{name}", obj) diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py new file mode 100644 index 000000000..30157f8bb --- /dev/null +++ b/tests/e2e/sre_authority/fixtures.py @@ -0,0 +1,226 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import io +import json +import tarfile + +from .common import ( + AGENT, CLAIM_VERSION, CONTEXT, FIELD_MANAGER, NAMESPACE_UID, OPERATORS, + PRIVATE, RUNTIME, SOURCE_NAME, SOURCE_NS, SOURCE_UID, STANDIN, SYSTEM, TENANT, + enrollment_json, fingerprint, require, +) +from .credential_paths import seed_privacy_gaps + +LEGACY_COMMIT = "8b206065608593667a40665b3f48225ef9ce278d" +CONTROL = "e2e-control-rotation" +CONTROL_NS = f"kars-{CONTROL}" +GROUP_BINDING = "e2e-sre-legacy-group" + + +def namespace_claim(h, source, name): + namespace = h.get("namespace", name) + if not namespace: + namespace = h.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}}) + uid = namespace["metadata"]["uid"] + h.api("PATCH", f"/api/v1/namespaces/{name}", body={ + "metadata": {"uid": uid, "resourceVersion": namespace["metadata"]["resourceVersion"], + "annotations": {CLAIM_VERSION: "v1", SOURCE_NS: SYSTEM, + SOURCE_NAME: source["metadata"]["name"], SOURCE_UID: source["metadata"]["uid"]}}, + }, status=200) + h.k("patch", "karssandbox", source["metadata"]["name"], "-n", SYSTEM, "--type=merge", "-p", + json.dumps({"metadata": {"uid": source["metadata"]["uid"], + "annotations": {NAMESPACE_UID: uid}}})) + return uid + + +def service_account(h, name, namespace): + return h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": name, "namespace": namespace}}) + + +def role_binding(h, name, role, account, namespace=None, account_namespace=OPERATORS): + return h.create({"apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding" if namespace else "ClusterRoleBinding", + "metadata": {"name": name, **({"namespace": namespace} if namespace else {})}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": role}, + "subjects": [{"kind": "ServiceAccount", "name": account, "namespace": account_namespace}]}) + + +def prepare_legacy(h): + require(not h.get("validatingadmissionpolicy", "kars-sre-source-authority"), + "Legacy fixtures must precede the new SRE admission guards") + require(not h.get("namespace", RUNTIME), "Disposable cluster contains a pre-existing SRE namespace") + h.run(["git", "fetch", "--no-tags", "--depth=1", "https://github.com/Azure/kars.git", LEGACY_COMMIT], timeout=90) + # git archive is binary; invoke separately without decoding tar as text. + import subprocess + data = subprocess.run(["git", "archive", LEGACY_COMMIT, "deploy/helm/kars"], + cwd=h.root, capture_output=True, timeout=30, check=True).stdout + destination = h.work / "legacy-chart" + destination.mkdir(exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(data)) as chart: + for member in chart.getmembers(): + require((member.name.startswith("deploy/helm/kars/") + or member.name.rstrip("/") in ("deploy", "deploy/helm", "deploy/helm/kars")) + and ".." not in member.name.split("/") + and (member.isdir() or member.isfile()), "Unsafe historical chart archive") + chart.extractall(destination, filter="data") + h.run(["helm", "install", "kars", str(destination / "deploy/helm/kars"), + "--kube-context", CONTEXT, "--namespace", SYSTEM, "--create-namespace", + "--set", "controller.replicas=0", "--set", "controller.image.repository=kars-controller", + "--set", "controller.image.tag=e2e", "--set", "controller.image.pullPolicy=Never", + "--set", "inferenceRouter.image.repository=kars-inference-router", + "--set", "inferenceRouter.image.tag=e2e", "--set", "sandbox.image.repository=kars-sandbox-e2e", + "--set", "sandbox.image.tag=dev", "--set-string", f"runtimes.hermes.image={STANDIN}", + "--set", "sre.enabled=false", "--set-string", "inferenceRouter.azure.openai.endpoint=https://e2e-fake.invalid/", + "--set-string", "foundry.endpoint=https://e2e-fake.invalid/"], timeout=180) + h.k("wait", "--for=condition=Established", "crd/karssandboxes.kars.azure.com", "--timeout=60s", timeout=70) + h.run(["helm", "upgrade", "kars", str(destination / "deploy/helm/kars"), "--kube-context", CONTEXT, + "--namespace", SYSTEM, "--reuse-values", "--set", "sre.enabled=true"], timeout=120) + source = h.get("karssandbox", "sre", SYSTEM) + require(source is not None, "Historical chart did not create its source CR") + h.state["legacy_source_uid"] = source["metadata"]["uid"] + h.state["legacy_namespace_uid"] = namespace_claim(h, source, RUNTIME) + h.state["system_uid"] = h.get("namespace", SYSTEM)["metadata"]["uid"] + h.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": OPERATORS}}) + h.create({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": TENANT}}) + service_account(h, "sandbox", RUNTIME) + for account in ("registrar", "unrelated"): + service_account(h, account, OPERATORS) + for account in ("tenant", "normal"): + service_account(h, account, TENANT) + h.token_identity("old-agent", RUNTIME, "sandbox") + h.token_identity("unrelated", OPERATORS, "unrelated") + binding = h.get("clusterrolebinding", "kars-sre-reader") + require(binding and binding.get("subjects"), "Historical reader grant is absent") + unrelated = {"kind": "ServiceAccount", "name": "unrelated", "namespace": OPERATORS} + h.k("patch", "clusterrolebinding", "kars-sre-reader", "--type=merge", "-p", json.dumps({ + "metadata": {"uid": binding["metadata"]["uid"], "resourceVersion": binding["metadata"]["resourceVersion"]}, + "subjects": binding["subjects"] + [unrelated], + })) + h.state["unrelated_subject"] = unrelated + h.create({"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": "sre", "namespace": RUNTIME, "labels": { + "kars.azure.com/component": "sandbox", "kars.azure.com/sandbox": "sre", + "kars.azure.com/parent-namespace": SYSTEM}}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"kars.azure.com/sandbox": "sre"}}, + "template": {"metadata": {"labels": {"kars.azure.com/sandbox": "sre"}}, + "spec": {"serviceAccountName": "sandbox", + "securityContext": {"runAsUser": 1000, "runAsNonRoot": True, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "agent", "image": STANDIN, "imagePullPolicy": "IfNotPresent", + "command": ["/bin/sh", "-c", "echo legacy-consumer-fixture; sleep infinity"]}]}}}}) + h.k("rollout", "status", "deployment/sre", "-n", RUNTIME, "--timeout=120s", timeout=130) + seed_control_consumer(h) + h.state["legacy_review_before_guards"] = enrollment_json(h.cli("authority", "preview")) + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", + "metadata": {"name": GROUP_BINDING}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "kars-sre-reader"}, + "subjects": [{"kind": "Group", "name": "system:serviceaccounts:kars-sre", "apiGroup": "rbac.authorization.k8s.io"}]}) + seed_privacy_gaps(h) + require(not h.get("validatingadmissionpolicy", "kars-sre-source-authority"), + "Legacy grants were not seeded before admission") + # Bootstrap only the new cluster API, not any of its admission policies. + # This makes kubectl's real discovery usable by authority stage's can-i. + rendered = h.run(["helm", "template", "kars", str(h.root / "deploy/helm/kars"), + "--namespace", SYSTEM, "--show-only", "templates/crd-karssreregistration.yaml"]) + converter = "const y=require('node:module').createRequire(process.cwd()+'/cli/package.json')('yaml'),f=require('fs');console.log(JSON.stringify(y.parse(f.readFileSync(0,'utf8'))));" + obj = json.loads(h.run(["node", "-e", converter], data=rendered, timeout=20)) + obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" + obj["metadata"].setdefault("annotations", {}).update({ + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM}) + h.create(obj) + h.k("wait", "--for=condition=Established", "crd/karssreregistrations.kars.azure.com", "--timeout=60s", timeout=70) + before = h.get("clusterrolebinding", "kars-sre-reader") + h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", "--dry-run", timeout=150) + after = h.get("clusterrolebinding", "kars-sre-reader") + require(before["metadata"]["uid"] == after["metadata"]["uid"] and before["subjects"] == after["subjects"], + "Authority stage preview changed legacy grants") + h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", timeout=180) + h.save() + h.passed("Legacy source/grants/consumer seeded using real UIDs before new policies; immutable old chart only, no old binary execution") + h.passed("Actual authority stage preview/apply retains legacy grants without private issuance") + + +def seed_control_consumer(h): + source = h.create({"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": CONTROL, "namespace": SYSTEM}, + "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1", + "command": ["/bin/sh"], "args": ["-c", "sleep infinity"]}}, + "sandbox": {"isolation": "standard"}}}) + namespace_uid = namespace_claim(h, source, CONTROL_NS) + secret = h.create({"apiVersion": "v1", "kind": "Secret", + "metadata": {"name": "router-services-admin", "namespace": CONTROL_NS, + "labels": {"app.kubernetes.io/managed-by": "kars-controller"}, + "annotations": {SOURCE_UID: source["metadata"]["uid"], NAMESPACE_UID: namespace_uid}}, + "stringData": {"control-token": "legacy-exposed-control-fixture-" + "x" * 40}}) + h.state["control_digest"] = fingerprint(secret) + h.state["control_secret_uid"] = secret["metadata"]["uid"] + h.state["control_source_uid"] = source["metadata"]["uid"] + h.create({"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": CONTROL, "namespace": CONTROL_NS, "labels": { + "kars.azure.com/component": "sandbox", "kars.azure.com/sandbox": CONTROL, + "kars.azure.com/parent-namespace": SYSTEM}}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"kars.azure.com/sandbox": CONTROL}}, + "template": {"metadata": {"labels": {"kars.azure.com/sandbox": CONTROL}}, + "spec": {"securityContext": {"runAsUser": 1000, "runAsNonRoot": True, "fsGroup": 1000, + "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [{"name": "credential-consumer", "image": STANDIN, + "command": ["/bin/sh", "-c", "sha256sum /credential/control-token > /proof/cached.sha; echo control-consumer-fixture; sleep infinity"], + "volumeMounts": [{"name": "old-control", "mountPath": "/credential", "readOnly": True}, + {"name": "proof", "mountPath": "/proof"}]}], + "volumes": [{"name": "old-control", "secret": {"secretName": "router-services-admin"}}, + {"name": "proof", "emptyDir": {}}]}}}}, manager="sre-e2e-fixture") + # Proven controller ownership of spec, without taking ownership of the + # fixture's extra cache-observation sidecar or its volumes. + h.k("apply", "--server-side", f"--field-manager={FIELD_MANAGER}", "-f", "-", data=json.dumps({ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": CONTROL, "namespace": CONTROL_NS}, + "spec": {"replicas": 1}})) + h.k("rollout", "status", f"deployment/{CONTROL}", "-n", CONTROL_NS, "--timeout=120s", timeout=130) + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="old-agent", status=200) + h.passed("Held legacy agent principal can genuinely read an owned control credential before migration") + + +def delegate_operators(h): + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": "e2e-sre-review-reader"}, "rules": [ + {"apiGroups": [""], "resources": ["namespaces"], "verbs": ["get", "list"]}, + {"apiGroups": ["apps"], "resources": ["deployments"], "verbs": ["get", "list"]}, + {"apiGroups": ["kars.azure.com"], "resources": ["karssandboxes"], "verbs": ["get", "list"]}, + {"apiGroups": ["apiextensions.k8s.io"], "resources": ["customresourcedefinitions"], "verbs": ["get"]}, + {"apiGroups": ["rbac.authorization.k8s.io"], "resources": ["clusterrolebindings", "rolebindings", "clusterroles", "roles"], "verbs": ["get", "list"]}, + ]}) + role_binding(h, "e2e-sre-review-reader", "e2e-sre-review-reader", "registrar") + role_binding(h, "e2e-sre-registrar", "kars-sre-registrar", "registrar") + h.token_identity("registrar", OPERATORS, "registrar") + h.token_identity("tenant", TENANT, "tenant") + h.token_identity("normal", TENANT, "normal") + for namespace in (RUNTIME, TENANT): + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", + "metadata": {"name": "e2e-sre-admission-probe", "namespace": namespace}, "rules": [ + {"apiGroups": [""], "resources": ["pods", "pods/ephemeralcontainers", "pods/exec", "pods/attach", "pods/portforward", "pods/proxy", "serviceaccounts", "serviceaccounts/token", "secrets"], "verbs": ["create", "patch", "update"]}, + {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list"]}, + {"apiGroups": [""], "resources": ["pods/exec", "pods/attach", "pods/portforward", "pods/proxy"], "verbs": ["get"]}, + {"apiGroups": ["apps"], "resources": ["deployments", "replicasets", "statefulsets", "daemonsets"], "verbs": ["create", "patch", "update"]}, + {"apiGroups": ["batch"], "resources": ["jobs", "cronjobs"], "verbs": ["create"]}, + {"apiGroups": ["kars.azure.com"], "resources": ["karssandboxes", "karssreactions"], "verbs": ["create", "patch", "update"]}, + ]}) + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": "e2e-sre-admission-probe", "namespace": namespace}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "e2e-sre-admission-probe"}, + "subjects": [{"kind": "ServiceAccount", "name": "tenant", "namespace": TENANT}]}) + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": "e2e-sre-registration-probe"}, "rules": [ + {"apiGroups": ["kars.azure.com"], "resources": ["karssreregistrations"], "verbs": ["create", "patch", "update"]}]}) + role_binding(h, "e2e-sre-registration-probe", "e2e-sre-registration-probe", "tenant", account_namespace=TENANT) + # Port-forward/exec are normal deployment rights, separate from registrar use. + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": "e2e-sre-registrar-runtime", "namespace": RUNTIME}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "e2e-sre-admission-probe"}, + "subjects": [{"kind": "ServiceAccount", "name": "registrar", "namespace": OPERATORS}]}) + require(h.k("auth", "can-i", "use", "karssreregistrations.kars.azure.com/canonical", + user="registrar").strip() == "yes", "Delegated registrar cannot use enrollment") + result = h.k("auth", "can-i", "create", "clusterrolebindings", user="registrar", expected=None) + require(result.returncode == 1 and result.stdout.strip() == "no", "Registrar fixture accidentally has cluster-admin-equivalent RBAC") + h.passed("Explicit registrar delegation works without granting cluster-admin to the registrar or any agent") diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py new file mode 100644 index 000000000..c6324cd13 --- /dev/null +++ b/tests/e2e/sre_authority/harness_test.py @@ -0,0 +1,207 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure harness checks. These are not the hosted Kubernetes acceptance result.""" + +import copy +import json +from pathlib import Path +import re +import tempfile +import types +import unittest +from unittest.mock import patch + +from sre_authority.common import ( + CLAIM_VERSION, NAMESPACE_UID, RUNTIME, SOURCE_NAME, SOURCE_NS, SOURCE_UID, SYSTEM, + Harness, assert_claim, assert_denial, enrollment_json, printed_object, review_args, +) +from sre_authority.proxy import MARKER, assert_filtered +from sre_authority.credential_paths import assert_watch_result + + +class Response: + def __init__(self, code, value): + self.status_code = code + self.value = value + + def json(self): + return self.value + + +class HarnessTests(unittest.TestCase): + def test_denials_require_real_forbidden_and_the_intended_policy(self): + valid = {"kind": "Status", "reason": "Forbidden", "message": "kars-sre-private-mounts denied"} + assert_denial(Response(403, valid), "probe", "kars-sre-private-mounts") + for code in (200, 201, 400, 404, 409, 422, 500): + with self.assertRaises(AssertionError): + assert_denial(Response(code, valid), "probe", "kars-sre-private-mounts") + with self.assertRaises(AssertionError): + assert_denial(Response(403, {**valid, "message": "ordinary RBAC denied"}), "probe", "kars-sre-private-mounts") + + def test_secret_watch_never_confuses_empty_success_with_authorization_denial(self): + forbidden = Response(403, {"kind": "Status", "reason": "Forbidden"}) + assert_watch_result(forbidden, False, False) + assert_watch_result(Response(200, {}), True, True) + for observed in (True, False): + with self.assertRaises(AssertionError): + assert_watch_result(Response(200, {}), observed, False) + with self.assertRaises(AssertionError): + assert_watch_result(Response(200, {}), False, True) + with self.assertRaises(AssertionError): + assert_watch_result(Response(404, {"kind": "Status", "reason": "NotFound"}), False, False) + + def test_review_arguments_pin_every_uid_resource_version_and_consumer(self): + spec = { + "controller": {}, "sandbox": {"uid": "source-uid"}, "runtimeNamespace": {"uid": "namespace-uid"}, + "legacyBindings": [ + {"kind": "ClusterRoleBinding", "name": "reader", "uid": "binding-a", "resourceVersion": "10"}, + {"kind": "RoleBinding", "namespace": "work", "name": "writer", "uid": "binding-b", "resourceVersion": "20"}, + ], + "legacyConsumer": {"uid": "consumer-uid", "resourceVersion": "30"}, + } + args = review_args(spec, {"metadata": {"uid": "registration-uid", "resourceVersion": "40"}}) + self.assertIn("ClusterRoleBinding//reader=binding-a@10", args) + self.assertIn("RoleBinding/work/writer=binding-b@20", args) + self.assertIn("consumer-uid@30", args) + self.assertEqual(args[-4:], ["--registration-uid", "registration-uid", "--resource-version", "40"]) + self.assertEqual(enrollment_json(json.dumps(spec) + "\n--binding 'review'\n"), spec) + with self.assertRaises(AssertionError): + enrollment_json('{"kind":"Status","reason":"Forbidden"}') + self.assertEqual(printed_object("Preview:\n" + json.dumps({"spec": spec}) + "\n"), + {"spec": spec}) + + def test_namespace_proof_independently_checks_full_claim_and_backlink(self): + source = {"metadata": {"name": "sre", "namespace": SYSTEM, "uid": "source-uid", "resourceVersion": "1", + "annotations": {NAMESPACE_UID: "namespace-uid"}}} + namespace = {"metadata": {"name": RUNTIME, "uid": "namespace-uid", "resourceVersion": "2", + "annotations": {CLAIM_VERSION: "v1", SOURCE_NS: SYSTEM, SOURCE_NAME: "sre", SOURCE_UID: "source-uid"}}} + core = {"metadata": {"uid": "core-uid"}} + assert_claim(source, namespace, core, "core-uid") + for key in (CLAIM_VERSION, SOURCE_NS, SOURCE_NAME, SOURCE_UID): + bad = copy.deepcopy(namespace) + bad["metadata"]["annotations"][key] = "wrong" + with self.assertRaises(AssertionError): + assert_claim(source, bad, core, "core-uid") + bad = copy.deepcopy(source) + bad["metadata"]["annotations"][NAMESPACE_UID] = "old-namespace-uid" + with self.assertRaises(AssertionError): + assert_claim(bad, namespace, core, "core-uid") + with self.assertRaises(AssertionError): + assert_claim(source, namespace, core, "old-core-uid") + + def test_real_api_status_handler_rejects_errors_without_echoing_secret_bodies(self): + harness = Harness.__new__(Harness) + seen = [] + def request(method, path, **kwargs): + seen.append((method, path, kwargs)) + return Response(403, {"message": MARKER}) + harness.client = lambda _user: types.SimpleNamespace(request=request) + with self.assertRaises(AssertionError) as failure: + harness.api("PATCH", "/api/v1/namespaces/example/secrets/example", + body={"stringData": {"token": MARKER}}, status=200) + self.assertIn("HTTP 403", str(failure.exception)) + self.assertNotIn(MARKER, str(failure.exception)) + self.assertEqual(seen[0][2]["headers"]["Content-Type"], "application/merge-patch+json") + self.assertEqual(harness.api("GET", "/missing").status_code, 403) + + def test_secret_projection_assertions_detect_all_copy_channels(self): + safe = {"kind": "Secret", "metadata": {"name": "example"}, + "data": {"operator-token": "", "password": ""}} + assert_filtered(safe) + for key in ("labels", "annotations", "managedFields"): + with self.assertRaises(AssertionError): + assert_filtered({**safe, "metadata": {"name": "example", key: {"copy": MARKER}}}) + with self.assertRaises(AssertionError): + assert_filtered({**safe, "stringData": {"copy": MARKER}}) + with self.assertRaises(AssertionError): + assert_filtered({**safe, "data": {"operator-token": MARKER, "password": ""}}) + + def test_token_probe_cannot_retain_admin_client_certificate(self): + root = Path(__file__).resolve().parent + with tempfile.TemporaryDirectory(prefix=".harness-unit-", dir=root) as folder: + harness = Harness.__new__(Harness) + harness.work = Path(folder) + harness.config = { + "clusters": [{"name": "kind-kars-e2e", "cluster": {"server": "https://127.0.0.1", "certificate-authority-data": "Y2E="}}], + "users": [{"name": "admin", "user": {"client-certificate-data": "PRIVATE", "client-key-data": "PRIVATE"}}], + } + harness.clients = {} + harness.server = "https://127.0.0.1" + harness.token_config("opaque", "not-a-real-token") + config = json.loads((harness.work / "opaque.json").read_text()) + self.assertEqual(config["users"][0]["user"], {"token": "not-a-real-token"}) + + class Context: + def __init__(self): + self.loaded = False + + def load_cert_chain(self, *_args): + self.loaded = True + + contexts = [] + def context_factory(**_kwargs): + context = Context() + contexts.append(context) + return context + clients = [] + def client_factory(**kwargs): + client = types.SimpleNamespace(**kwargs) + clients.append(client) + return client + harness.httpx = types.SimpleNamespace(Client=client_factory) + with patch("sre_authority.common.ssl.create_default_context", side_effect=context_factory): + harness.client("opaque") + harness.client("admin") + self.assertFalse(contexts[0].loaded) + self.assertTrue(contexts[1].loaded) + self.assertIsNot(contexts[0], contexts[1]) + self.assertNotIn("cert", clients[0].__dict__) + self.assertEqual(clients[0].headers["Authorization"], "Bearer not-a-real-token") + self.assertNotIn("Authorization", clients[1].headers) + + def test_legacy_seed_precedes_new_install_and_other_acceptance(self): + root = Path(__file__).resolve().parents[3] + main = (root / "tests/e2e/run.sh").read_text().split("main() {", 1)[1] + self.assertLess(main.index("prepare_sre_authority_legacy"), main.index("install_crds")) + self.assertLess(main.index("test_sre_authority_migration"), main.index("test_create_sandbox")) + fixture = (root / "tests/e2e/sre_authority/fixtures.py").read_text() + self.assertIn("8b206065608593667a40665b3f48225ef9ce278d", fixture) + self.assertIn("https://github.com/Azure/kars.git", fixture) + self.assertNotIn("--force-conflicts", fixture) + self.assertNotIn("--validate=false", fixture) + self.assertLess(fixture.index('h.state["legacy_review_before_guards"]'), + fixture.index("seed_privacy_gaps(h)")) + self.assertLess(fixture.index("seed_privacy_gaps(h)"), + fixture.index('h.cli("authority", "stage"')) + from sre_authority.common import POLICIES + self.assertIn("no-legacy-tokens", POLICIES) + + def test_kind_prerequisites_do_not_remove_the_rust_httpx_test_step(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + self.assertIn("Legacy Hermes HTTPS client test dependency", workflow) + kind = workflow.split(" e2e-kind:", 1)[1].split(" bench-regression:", 1)[0] + self.assertIn("actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", kind) + self.assertIn("npm ci && npm run build", kind) + self.assertIn("'httpx==0.28.1'", kind) + self.assertGreaterEqual(workflow.count("'httpx==0.28.1'"), 2) + + def test_kind_runtime_filter_includes_all_shared_security_modules(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + kind = workflow.split(" e2e-kind:", 1)[1].split(" bench-regression:", 1)[0] + expression = re.search(r"\| grep -E '([^']+)'", kind) + self.assertIsNotNone(expression) + pattern = expression.group(1) + for path in ("shared/sre_privacy.rs", "shared/another_security_module.rs", + "controller/src/sre_authority.rs", "cli/src/lib/sre-authority.ts"): + with self.subTest(path=path): + self.assertRegex(path, pattern) + for path in ("docs/how-to/sre-authority.md", "unrelated/shared/sre_privacy.rs"): + with self.subTest(path=path): + self.assertNotRegex(path, pattern) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/migration.py b/tests/e2e/sre_authority/migration.py new file mode 100644 index 000000000..9b0a2a342 --- /dev/null +++ b/tests/e2e/sre_authority/migration.py @@ -0,0 +1,252 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json + +from .common import ( + AGENT, EPOCH, OPERATORS, PRIVATE, RUNTIME, SYSTEM, TENANT, + assert_claim, assert_denial, enrollment_json, fingerprint, printed_object, require, review_args, +) +from .fixtures import CONTROL, CONTROL_NS, GROUP_BINDING, delegate_operators +from .admission import admission_cases, policies_ready +from .credential_paths import block_prestaged_paths, fixture_review, secret_watch, token_secret_denials + + +def cli_failure(result, text, label): + require(result.returncode != 0 and text in (result.stdout + result.stderr), + f"{label}: command did not fail for the intended reason") + + +def pods(h, namespace): + return json.loads(h.k("get", "pods", "-n", namespace, "-o", "json"))["items"] + + +def snapshot_grants(h, spec): + return {binding["name"]: h.get(binding["kind"].lower(), binding["name"], binding.get("namespace")) + for binding in spec["legacyBindings"]} + + +def legacy_migration(h): + policies_ready(h) + delegate_operators(h) + # Group grants were deliberately created before admission, never smuggled + # through the new deny policies. + original = h.get("clusterrolebinding", "kars-sre-reader") + cli_failure(h.cli("authority", "preview", user="registrar", expected=None), + "broad group grant", "broad-group review") + require(h.get("clusterrolebinding", "kars-sre-reader")["subjects"] == original["subjects"], + "Group ambiguity mutated direct legacy subjects") + require(h.get("secret", PRIVATE, RUNTIME) is None, "Private material appeared before explicit enrollment") + h.k("delete", "clusterrolebinding", GROUP_BINDING, "--wait=true", "--timeout=30s") + h.passed("Actual preview blocks an unsafe pre-existing group grant before direct grants or credentials change") + cli_failure(h.cli("authority", "preview", user="registrar", expected=None), + "broad group grant", "watch-only broad-group review") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", + user="watch-only"), "watch-only principal GET") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets", + user="watch-only"), "watch-only principal LIST") + secret_watch(h, CONTROL_NS, "e2e-watch-only-before", user="watch-only", allowed=True) + h.passed("Watch-only group member is denied Secret GET/LIST but genuinely observes a newly created dummy Secret; preview blocks the grant") + + h.k("rollout", "status", f"deployment/{CONTROL}", "-n", CONTROL_NS, "--timeout=150s", timeout=160) + spec = fixture_review(h) + require(spec["sandbox"]["uid"] == h.state["legacy_source_uid"], "Staging adopted a replacement SRE source") + require(spec["runtimeNamespace"]["uid"] == h.state["legacy_namespace_uid"], "Staging replaced the runtime namespace") + assert_claim(h.get("karssandbox", "sre", SYSTEM), h.get("namespace", RUNTIME), + h.get("namespace", SYSTEM), h.state["system_uid"]) + require(spec.get("legacyConsumer") and len(spec["legacyBindings"]) >= 2, "Legacy reviews are incomplete") + h.state["old_agent_sa_uid"] = h.get("serviceaccount", "sandbox", RUNTIME)["metadata"]["uid"] + old_pods = {pod["metadata"]["uid"] for pod in pods(h, RUNTIME)} + old_control_pods = {pod["metadata"]["uid"] for pod in pods(h, CONTROL_NS)} + require(old_pods and old_control_pods, "Legacy/control consumers are not actually running") + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="old-agent", status=200) + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets", user="old-agent", status=200) + before = snapshot_grants(h, spec) + admission_cases(h, spec) + token_secret_denials(h, before_enrollment=True) + # The controller rejects the pre-existing token alias first, then the + # watch-only group grant. Only explicit UID/RV-fenced fixture cleanup + # removes them; the registration is left blocked on an incomplete review. + blocked = block_prestaged_paths(h, spec, before) + cli_failure(h.cli("authority", "enroll", "--sandbox-uid", spec["sandbox"]["uid"], + "--namespace-uid", spec["runtimeNamespace"]["uid"], + user="registrar", expected=None), "Every legacy binding", "missing binding reviews") + current = h.get("karssreregistrations.kars.azure.com", "canonical") + require(current["metadata"]["uid"] == blocked["metadata"]["uid"] and current["spec"] == blocked["spec"] + and current["metadata"]["generation"] == blocked["metadata"]["generation"], + "Unreviewed CLI enrollment changed the blocked registration") + require(snapshot_grants(h, spec) == before, "Unreviewed enrollment mutated legacy bindings") + require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, "Unreviewed enrollment stopped the consumer") + h.passed("Unreviewed bindings/consumer cannot enroll or mutate an occupied legacy source") + require(snapshot_grants(h, spec) == before, "Controller mutated grants before validating the entire review set") + require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, "Controller stopped an unreviewed consumer") + require(h.get("secret", PRIVATE, RUNTIME) is None, "Controller issued private material for an incomplete review") + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="old-agent", status=200) + h.passed("Real registration/controller rejects an incomplete review before grant retirement, consumer stop or private issuance") + + # Refresh once after read-only tests; use the exact current RVs, not guessed + # names or stale values from the fixture bootstrap. + spec = enrollment_json(h.cli("authority", "preview", user="registrar")) + reviewed = review_args(spec, current) + preview = h.cli("authority", "enroll", *reviewed, "--dry-run", user="registrar") + require(printed_object(preview)["spec"] == spec, "Enrollment dry-run differs from reviewed API identities") + after = h.get("karssreregistrations.kars.azure.com", "canonical") + require(after["spec"] == current["spec"] and after["metadata"]["generation"] == current["metadata"]["generation"] + and after["metadata"]["uid"] == current["metadata"]["uid"], "Enrollment dry-run mutated registration") + reviewed = review_args(spec, after) + h.cli("authority", "enroll", *reviewed, user="registrar") + require(h.get("karssreregistrations.kars.azure.com", "canonical")["spec"] == spec, + "Persisted enrollment differs from the exact reviewed identities") + + issuance_seen = False + def migrated(): + nonlocal issuance_seen + private = h.get("secret", PRIVATE, RUNTIME) + if private and "kube-token" in private.get("data", {}) and not issuance_seen: + # Real held bearer token, with no client certificate fallback. + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", + user="old-agent"), "old held principal GET before private issuance") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets", + user="old-agent"), "old held principal LIST before private issuance") + assert_denial(h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets?watch=true&timeoutSeconds=1", + user="old-agent"), "old held principal WATCH before private issuance") + require(not old_pods.intersection(pod["metadata"]["uid"] for pod in pods(h, RUNTIME)), + "Private credentials issued while a reviewed old consumer still exists") + issuance_seen = True + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + require(reg is not None, "Enrollment disappeared during migration") + status = reg.get("status", {}) + require(not (status.get("phase") == "Blocked" and status.get("observedGeneration") == reg["metadata"]["generation"]), + "Reviewed SRE migration was blocked") + return reg if status.get("phase") == "Ready" and status.get("observedGeneration") == reg["metadata"]["generation"] else False + reg = h.poll("reviewed migration and credential-order proof", migrated, seconds=240, interval=0.25) + require(issuance_seen, "No private issuance was observed") + h.cli("authority", "migrate", user="registrar", timeout=45) + h.wait_ready() + h.passed("Real old-principal GET/LIST/WATCH denial and old Pod termination precede private credential issuance") + for namespace, cluster, name in [ + (CONTROL_NS, False, "e2e-old-watch-control"), + (RUNTIME, False, "e2e-old-watch-private"), + (CONTROL_NS, True, "e2e-old-watch-cluster"), + ]: + secret_watch(h, namespace, name, cluster=cluster) + h.passed("Held legacy token receives actual HTTP Forbidden for namespaced and cluster Secret watches, including newly created dummy-Secret probes") + require(h.get("serviceaccount", "sandbox", RUNTIME)["metadata"]["uid"] == h.state["old_agent_sa_uid"], + "Migration used ServiceAccount recreation instead of revoking held-principal authorization") + reader = h.get("clusterrolebinding", "kars-sre-reader") + require(h.state["unrelated_subject"] in reader["subjects"], "Migration removed an unrelated binding subject") + require(not any(subject.get("name") == "sandbox" and subject.get("namespace") == RUNTIME for subject in reader["subjects"]), + "Legacy SRE subject remains bound") + h.api("GET", f"/api/v1/namespaces/{CONTROL_NS}/secrets/router-services-admin", user="unrelated", status=200) + h.passed("Unrelated binding subjects remain authorized; the old Sandbox ServiceAccount UID is preserved but denied") + + rotated = h.get("secret", "router-services-admin", CONTROL_NS) + require(rotated["metadata"]["uid"] == h.state["control_secret_uid"], "Owned control Secret was replaced instead of rotated") + require(fingerprint(rotated) != h.state["control_digest"], "Owned exposed control token was not rotated") + current_pods = pods(h, CONTROL_NS) + require(current_pods and not old_control_pods.intersection(pod["metadata"]["uid"] for pod in current_pods), + "A startup-cached old control-token consumer survived Ready") + expected = fingerprint(rotated) + for pod in current_pods: + require(pod["metadata"].get("annotations", {}).get(EPOCH) == reg["status"]["privacyEpoch"], + "Control consumer lacks the new privacy epoch") + cached = h.k("exec", "-n", CONTROL_NS, pod["metadata"]["name"], "-c", "credential-consumer", + "--", "cat", "/proof/cached.sha", timeout=25).split()[0] + require(cached == expected, "Restarted control consumer did not cache the rotated token") + h.passed("Owned router-services-admin token rotates and every cached old consumer is gone before Ready") + h.cli("install", timeout=240) + rollback_guard(h) + h.state["legacy_registration_uid"] = reg["metadata"]["uid"] + h.save() + + +def rollback_guard(h): + # Execute the actual CLI guard against this API, without pretending a Kind + # cluster has an Azure upgrade context or mocking an Azure command. + script = """ +const {pathToFileURL}=require('node:url'); +const {execa}=await import(pathToFileURL(process.cwd()+'/cli/node_modules/execa/index.js')); +const {assertRollbackSafe}=await import(pathToFileURL(process.cwd()+'/cli/dist/lib/sre-authority.js')); +const execute=(file,args,options)=>execa(file,['--context','kind-kars-e2e',...args],options); +try { await assertRollbackSafe(execute); process.exitCode=9; } +catch(error) { if(!String(error.message).includes('Rollback across enrolled SRE authority is unsafe')) throw error; } +""" + # CommonJS eval cannot use top-level await; the async wrapper keeps imports + # on the actual compiled CLI module, not an alternate acceptance model. + h.run(["node", "-e", f"(async()=>{{{script}}})().catch(()=>{{process.exitCode=1}})"], timeout=30) + h.passed("Actual CLI rollback guard rejects regranting a legacy release while enrollment audit history exists") + + +def retire_and_uninstall(h): + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + h.cli("authority", "retire", "--registration-uid", reg["metadata"]["uid"], + "--resource-version", reg["metadata"]["resourceVersion"], user="registrar", timeout=240) + retired = h.get("karssreregistrations.kars.azure.com", "canonical") + require(retired["status"]["phase"] == "Retired" and retired["spec"]["enabled"] is False, + "Retirement did not complete") + for name in (PRIVATE, AGENT): + require(h.get("secret", name, RUNTIME) is None, "Private credential survived retirement") + for kind in ("clusterrolebindings", "rolebindings"): + items = json.loads(h.k("get", kind, "-A", "-o", "json"))["items"] + require(not any(item["metadata"].get("annotations", {}).get("kars.azure.com/sre-registration-uid") == reg["metadata"]["uid"] + for item in items), "Owned private grant survived retirement") + h.cli("uninstall", timeout=180) + h.poll("SRE source removal", lambda: h.get("karssandbox", "sre", SYSTEM) is None, seconds=120) + h.poll("SRE namespace removal", lambda: h.get("namespace", RUNTIME) is None, seconds=120) + require(h.get("namespace", SYSTEM)["metadata"]["uid"] == h.state["system_uid"], "Retirement changed core namespace identity") + retained = h.get("karssreregistrations.kars.azure.com", "canonical") + require(retained["metadata"]["uid"] == reg["metadata"]["uid"] and retained["status"]["phase"] == "Retired", + "Retired registration audit record was removed or replaced") + h.passed("Retire/uninstall revoke owned grants/credentials and remove the source namespace while retaining core UID and audit registration") + + +def fresh_reenrollment(h): + policies_ready(h) + retained = h.get("karssreregistrations.kars.azure.com", "canonical") + require(retained and retained["status"]["phase"] == "Retired", "Fresh phase requires retained retired history") + foreign = h.create({"apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": RUNTIME, "labels": {"e2e-foreign-occupant": "true"}}}) + h.create({"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "foreign-sentinel", "namespace": RUNTIME}, + "data": {"value": "must-remain"}}) + cli_failure(h.cli("authority", "stage-source", expected=None), + "Existing kars-sre namespace", "foreign occupant") + require(h.get("namespace", RUNTIME)["metadata"]["uid"] == foreign["metadata"]["uid"], "Foreign namespace was replaced") + require(h.get("configmap", "foreign-sentinel", RUNTIME)["data"]["value"] == "must-remain", "Foreign content was mutated") + require(h.get("karssandbox", "sre", SYSTEM) is None and h.get("secret", PRIVATE, RUNTIME) is None, + "Foreign occupancy acquired a source or private privilege") + h.namespace_delete(RUNTIME, foreign["metadata"]["uid"]) + h.passed("Foreign runtime occupancy blocks fresh staging before source creation or private issuance") + + created = h.cli("authority", "stage-source") + spec = enrollment_json(h.cli("authority", "preview", user="registrar")) + assert_claim(h.get("karssandbox", "sre", SYSTEM), h.get("namespace", RUNTIME), + h.get("namespace", SYSTEM), h.state["system_uid"]) + require(f'--sandbox-uid {spec["sandbox"]["uid"]}' in created + and f'--namespace-uid {spec["runtimeNamespace"]["uid"]}' in created, + "Fresh staging did not return the actual CREATE/claim UIDs") + require(spec["sandbox"]["uid"] != h.state["legacy_source_uid"] and + spec["runtimeNamespace"]["uid"] != h.state["legacy_namespace_uid"], "Fresh source reused old authority identities") + cli_failure(h.cli("authority", "stage-source", expected=None), "SRE source already exists", "source adoption race") + cli_failure(h.cli("authority", "enroll", "--sandbox-uid", h.state["legacy_source_uid"], + "--namespace-uid", h.state["legacy_namespace_uid"], user="registrar", expected=None), + "no longer matches", "stale source UID enrollment") + retained = h.get("karssreregistrations.kars.azure.com", "canonical") + reviewed = review_args(spec, retained) + dry = printed_object(h.cli("authority", "enroll", *reviewed, "--dry-run", user="registrar")) + require(dry["spec"] == spec, "Fresh enrollment differs from its reviewed identities") + h.cli("authority", "enroll", *reviewed, user="registrar") + require(h.get("karssreregistrations.kars.azure.com", "canonical")["spec"] == spec, + "Fresh CAS re-enrollment retained unreviewed identities from the retired registration") + h.cli("authority", "migrate", user="registrar", timeout=240) + h.wait_ready() + h.cli("install", timeout=240) + # Recreate only our test Role/Bindings in the newly claimed namespace. + role = h.get("role", "e2e-sre-admission-probe", TENANT) + h.create({"apiVersion": role["apiVersion"], "kind": "Role", + "metadata": {"name": role["metadata"]["name"], "namespace": RUNTIME}, "rules": role["rules"]}) + for account, namespace in (("tenant", TENANT), ("registrar", OPERATORS)): + h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": f"e2e-sre-{account}-runtime", "namespace": RUNTIME}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": role["metadata"]["name"]}, + "subjects": [{"kind": "ServiceAccount", "name": account, "namespace": namespace}]}) + h.passed("Fresh atomic source/claim and delegated CAS re-enrollment reach Ready without adopting stale UIDs") diff --git a/tests/e2e/sre_authority/proxy.py b/tests/e2e/sre_authority/proxy.py new file mode 100644 index 000000000..f39eb6fca --- /dev/null +++ b/tests/e2e/sre_authority/proxy.py @@ -0,0 +1,206 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import importlib.util +import json +import os +import sys +import types + +from .common import AGENT, EPOCH, OPERATORS, PRIVATE, RUNTIME, STANDIN, SYSTEM, require +from .admission import runtime_denials +from .credential_paths import token_secret_denials + +MARKER = "e2e-secret-value-must-not-cross-filter" +SA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount" + + +def install_metrics(h): + # Real metrics-server on the disposable Kind cluster. This is not a fake + # metrics API and does not change any production chart or SRE policy. + if not h.get("apiservice", "v1beta1.metrics.k8s.io"): + h.k("apply", "-f", "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml", + timeout=90) + h.k("patch", "deployment", "metrics-server", "-n", "kube-system", "--type=json", "-p", + json.dumps([{"op": "add", "path": "/spec/template/spec/containers/0/args/-", + "value": "--kubelet-insecure-tls"}])) + h.k("rollout", "status", "deployment/metrics-server", "-n", "kube-system", "--timeout=120s", timeout=130) + def collected(): + response = h.api("GET", "/apis/metrics.k8s.io/v1beta1/nodes") + return response.status_code == 200 and bool(response.json().get("items")) + h.poll("real Kind node metrics", collected, seconds=100, interval=3) + + +def alive_pinned_source(h): + source = h.get("karssandbox", "sre", SYSTEM) + uid = source["metadata"]["uid"] + # An explicit, valid BYO image pin exercises projection preservation. This + # is a sleeping test consumer, not an old binary or a live model/agent run. + h.k("patch", "karssandbox", "sre", "-n", SYSTEM, "--type=merge", "-p", json.dumps({ + "metadata": {"uid": uid, "resourceVersion": source["metadata"]["resourceVersion"]}, + "spec": {"agent": None, "runtime": {"kind": "BYO", "hermes": None, + "byo": {"image": STANDIN, "contractVersion": "v1", "command": ["/bin/sh"], + "args": ["-c", "echo sre-standin-alive; sleep infinity"]}}}})) + def current(): + deployment = h.get("deployment", "sre", RUNTIME) + if not deployment: + return False + template = deployment["spec"]["template"]["spec"] + agent = next((item for item in template["containers"] if item["name"] == "agent"), {}) + runtime = next((entry.get("value") for entry in agent.get("env", []) if entry["name"] == "KARS_RUNTIME_KIND"), None) + return deployment if agent.get("image") == STANDIN and runtime == "BYO" else False + h.poll("registered BYO stand-in projection", current, seconds=150) + h.k("rollout", "status", "deployment/sre", "-n", RUNTIME, "--timeout=150s", timeout=160) + def ready(): + pods = json.loads(h.k("get", "pods", "-n", RUNTIME, "-l", "kars.azure.com/sandbox=sre", "-o", "json"))["items"] + live = [pod for pod in pods if not pod["metadata"].get("deletionTimestamp") + and any(condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in pod.get("status", {}).get("conditions", []))] + return live[0] if len(live) == 1 else False + pod = h.poll("private TLS router and alive stand-in Pod Ready", ready, seconds=100) + require(h.get("karssandbox", "sre", SYSTEM)["metadata"]["uid"] == uid, "Runtime pin replaced the registered source") + return pod + + +def projection_and_files(h, pod): + spec = pod["spec"] + require(spec["serviceAccountName"] == "sandbox", "Projection changed Azure's federated ServiceAccount subject") + require(spec.get("automountServiceAccountToken") is False, "Pod still auto-mounts a Kubernetes JWT") + registration = h.get("karssreregistrations.kars.azure.com", "canonical") + router_sa = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(router_sa and router_sa.get("automountServiceAccountToken") is False + and router_sa["metadata"]["uid"] == registration["status"]["routerServiceAccountUid"], + "Private router identity is not the registered non-automounting ServiceAccount") + agent = next(item for item in spec["containers"] if item["name"] == "agent") + router = next(item for item in spec["containers"] if item["name"] == "inference-router") + require(agent["image"] == STANDIN and router["image"] == "kars-inference-router:e2e", "Selected image pin/artifact changed") + agent_mounts = {mount["name"]: mount["mountPath"] for mount in agent["volumeMounts"]} + router_mounts = {mount["name"]: mount["mountPath"] for mount in router["volumeMounts"]} + require(agent_mounts.get("sre-api-agent") == SA_PATH, "Agent standard paths are not projected from the safe identity") + require("sre-api-private" not in agent_mounts and "router-kubernetes" not in agent_mounts + and "azure-identity-token" not in agent_mounts, "Agent received a private or projected Azure/Kubernetes credential") + require(router_mounts.get("sre-api-private") == "/etc/kars/sre-api" + and router_mounts.get("router-kubernetes") == SA_PATH, "Router lost its private/API credential isolation") + volumes = {volume["name"]: volume for volume in spec["volumes"]} + require(volumes["sre-api-agent"]["secret"]["secretName"] == AGENT, "Agent mapping references wrong Secret") + require(volumes["sre-api-private"]["secret"]["secretName"] == PRIVATE, "Router mapping references wrong Secret") + require({item["key"] for item in volumes["sre-api-agent"]["secret"]["items"]} == {"token", "ca.crt", "namespace"}, + "Agent safe projection exposes unexpected keys") + env = {item["name"]: item.get("value") for item in agent["env"]} + require(env.get("KUBERNETES_SERVICE_HOST") == "127.0.0.1" + and env.get("KUBERNETES_SERVICE_PORT") == "9446", "Legacy HTTPS endpoint is not the local private proxy") + skipped = pod["metadata"].get("annotations", {}).get("azure.workload.identity/skip-containers", "").split(",") + require("agent" in skipped and "egress-guard" in skipped, "Azure token injection is not excluded from the agent") + for name in ("token", "ca.crt", "namespace"): + # Only safe agent files are copied. Private credentials are never + # copied to the host, printed or placed in argv. + value = h.k("exec", "-n", RUNTIME, pod["metadata"]["name"], "-c", "agent", + "--", "cat", f"{SA_PATH}/{name}", user="registrar", timeout=25) + h.write(f"agent/{name}", value) + require((h.work / "agent/namespace").read_text().strip() == RUNTIME, "Standard namespace file changed") + token = (h.work / "agent/token").read_text().strip() + h.token_config("opaque", token) + response = h.api("GET", "/api/v1/namespaces", user="opaque") + require(response.status_code == 401, "Opaque agent credential authenticated to the real Kubernetes API") + review = h.api("POST", "/apis/authentication.k8s.io/v1/tokenreviews", body={ + "apiVersion": "authentication.k8s.io/v1", "kind": "TokenReview", "spec": {"token": token}}, + status=201).json() + require(review["status"].get("authenticated") is not True, "TokenReview accepted the opaque agent credential") + h.passed("Real Pod preserves Sandbox/Azure identity and image pin; only safe standard files reach the agent") + h.passed("Opaque agent credential is rejected by both real API authentication and TokenReview without admin-cert fallback") + + +def load_unchanged_hermes(h, port): + root = h.root / "runtimes/hermes/src/kars_runtime_hermes/plugin" + package = "_kars_e2e_unchanged_sre" + module = types.ModuleType(package) + module.__path__ = [str(root)] + sys.modules[package] = module + loaded = {} + for name in ("sre_kube", "sre"): + spec = importlib.util.spec_from_file_location(f"{package}.{name}", root / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + loaded[name] = module + # Test-only host relocation of the unchanged client's standard file root. + loaded["sre_kube"]._SA_DIR = h.work / "agent" + os.environ["KUBERNETES_SERVICE_HOST"] = "127.0.0.1" + os.environ["KUBERNETES_SERVICE_PORT"] = str(port) + os.environ["NO_PROXY"] = "127.0.0.1,localhost" + return loaded["sre_kube"], loaded["sre"] + + +def assert_filtered(secret): + require(secret.get("kind") == "Secret", "Filtered GET did not return a Secret projection") + require(set(secret.get("data", {})) == {"operator-token", "password"}, "Secret key names were lost") + require(all(value == "" for value in secret["data"].values()), "Secret values crossed the proxy") + require("stringData" not in secret, "stringData crossed the proxy") + for key in ("annotations", "labels", "managedFields"): + require(key not in secret["metadata"], f"Secret {key} copies crossed the proxy") + require(MARKER not in json.dumps(secret), "Secret material survived projection") + + +def proxy_acceptance(h): + install_metrics(h) + pod = alive_pinned_source(h) + projection_and_files(h, pod) + runtime_denials(h, pod["metadata"]["name"]) + token_secret_denials(h) + h.create({"apiVersion": "v1", "kind": "Secret", + "metadata": {"name": f"sre-filter-{h.phase}", "namespace": OPERATORS, + "labels": {"copy": MARKER}, + "annotations": {"kubectl.kubernetes.io/last-applied-configuration": json.dumps({"data": {"copy": MARKER}})}}, + "stringData": {"operator-token": MARKER, "password": MARKER}}) + name = f"sre-filter-{h.phase}" + with h.port_forward(pod["metadata"]["name"]) as port: + kube_module, sre = load_unchanged_hermes(h, port) + kube = kube_module.client() + try: + secret = kube.get(f"/api/v1/namespaces/{OPERATORS}/secrets/{name}") + assert_filtered(secret) + listing = kube.get(f"/api/v1/namespaces/{OPERATORS}/secrets", params={"fieldSelector": f"metadata.name={name}"}) + require(listing.get("kind") == "SecretList" and len(listing["items"]) == 1, "Filtered LIST did not reach the real API") + assert_filtered(listing["items"][0]) + h.passed("Unchanged Hermes HTTPS client GET/LIST retains Secret key names but no values or metadata copies") + logs = sre._impl_sre_logs(namespace=RUNTIME, pod=pod["metadata"]["name"], container="agent", tail=20) + require("error" not in logs and "sre-standin-alive" in logs.get("logs", ""), "Unchanged Hermes raw log reader failed") + metrics = kube.get("/apis/metrics.k8s.io/v1beta1/nodes") + require(metrics.get("kind") == "NodeMetricsList" and metrics.get("items"), "Real metrics did not pass the filtered proxy") + action = sre._create_karssreaction_cr( + action={"type": "RolloutRestart", "namespace": SYSTEM, "kind": "Deployment", "name": "kars-controller"}, + diagnosis="Kind acceptance only", rationale="Pending proposal; never approve or execute", ttl_minutes=5) + created = h.get("karssreaction", action, RUNTIME) + require(created and created["spec"]["approval"] == {"state": "Pending"}, "Real Hermes proposal did not persist Pending") + h.passed("Actual logs, real metrics and unchanged Hermes Pending proposal builder work over Pod TLS; no model run claimed") + raw = kube._ensure_client() + base = f"https://127.0.0.1:{port}" + for path in [ + "/api/v1/namespaces/%6bars-sre/pods", + "/api/v1/namespaces/kars-sre/pods/../secrets", + "/api/v1/namespaces/kars-sre/pods?watch=true", + f"/api/v1/namespaces/kars-sre/pods/{pod['metadata']['name']}/proxy", + f"/api/v1/namespaces/kars-sre/pods/{pod['metadata']['name']}/log?follow=true", + "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token", + ]: + # Encode dot segments explicitly so the HTTP client cannot + # normalize an escape into a different legitimate request. + path = path.replace("/../", "/%2e%2e/") + require(raw.get(base + path).status_code == 403, "Unsafe proxy path/query was not rejected") + target = f"/apis/kars.azure.com/v1alpha1/namespaces/{RUNTIME}/karssreactions/{action}" + require(raw.patch(base + target, json={"spec": {"approval": {"state": "Approved"}}}).status_code == 403, + "Self-approval PATCH was not rejected") + require(raw.post(base + f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router/token", + json={"spec": {"audiences": [], "expirationSeconds": 600}}).status_code == 403, + "Token creation escaped the filtered proxy") + require(raw.post(base + f"/api/v1/namespaces/{RUNTIME}/configmaps", + json={"metadata": {"name": "must-not-create"}}).status_code == 403, + "Arbitrary writes escaped the filtered proxy") + invalid = {**created, "metadata": {"name": "self-approved", "namespace": RUNTIME}, + "spec": {**created["spec"], "approval": {"state": "Approved"}}} + invalid.pop("status", None) + require(raw.post(base + f"/apis/kars.azure.com/v1alpha1/namespaces/{RUNTIME}/karssreactions", + json=invalid).status_code == 403, "Approved proposal creation was not rejected") + h.passed("Encoded/proxy/watch/token/write/self-approval paths are rejected, not mistaken for missing API resources") + finally: + kube.close() From 05b3a77c350bcf22465dadcc941829773e1b756c Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 10:38:58 +0200 Subject: [PATCH 02/62] fix(cli): reject invalid push targets before SRE preflight Restore the external-mesh no-probe rejection contract while keeping authority preflight ahead of artifact resolution and every deployment write. Cover a valid core target whose SRE preflight fails without mutation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/commands/push-apply.test.ts | 8 ++++++++ cli/src/commands/push-apply.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/src/commands/push-apply.test.ts b/cli/src/commands/push-apply.test.ts index 5b4388bc5..079c23e37 100644 --- a/cli/src/commands/push-apply.test.ts +++ b/cli/src/commands/push-apply.test.ts @@ -146,6 +146,14 @@ function fixture(options: { legacyCore?: boolean; mesh?: "absent" | "helm" | "ex } describe("selected core push artifacts", () => { + it("still requires SRE authority preflight before resolving or applying valid core artifacts", async () => { + const f = fixture({ fail: "karssreregistrations.kars.azure.com" }); + await expect(applyPushedImages(f.execute, [pushed("controller")], "chart")) + .rejects.toThrow("karssreregistrations.kars.azure.com"); + expect(f.calls().some(([bin, args]) => bin === "az" + || ["upgrade", "patch", "annotate", "rollout"].includes(args[0]))).toBe(false); + }); + it("moves a GHCR/pinned Helm controller to its pushed ACR digest without resetting customer values", async () => { const f = fixture(); await applyPushedImages(f.execute, [pushed("controller")], "chart"); diff --git a/cli/src/commands/push-apply.ts b/cli/src/commands/push-apply.ts index baf0c142e..81202b82c 100644 --- a/cli/src/commands/push-apply.ts +++ b/cli/src/commands/push-apply.ts @@ -32,7 +32,6 @@ export async function applyPushedImages( const buildOnly = images.filter(item => item.name === "sandbox-base").map(item => item.name); const deployable = images.filter(item => item.name !== "sandbox-base"); if (!deployable.length) throw new Error("sandbox-base is build-only; no deployment was applied"); - await assertSafeMutation(execute); const isMesh = (item: PushedImage) => item.name === "relay" || item.name === "registry"; const selectedCore = deployable.filter(item => !isMesh(item)); const selectedMesh = deployable.some(isMesh); @@ -43,6 +42,7 @@ export async function applyPushedImages( throw new Error("External or absent AgentMesh cannot be updated; choose explicit core targets instead."); } if (core?.kind === "helm" && mesh) assertMeshReleaseConsistency(mesh, core.values); + await assertSafeMutation(execute); const artifacts = await resolvePushedArtifacts(execute, deployable); const coreImages = artifacts.filter(item => !isMesh(item)); const meshImages: MeshImages = {}; From 941d6c2b1308c7939c1c2c6e5b95d82b67bce0ac Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 12:28:50 +0200 Subject: [PATCH 03/62] fix(sre): complete status conventions and verify request boundaries Publish real standard conditions and CRD metadata, distinguish immutable Secret type rejection from admission-policy denial, and make CLI fixture resource matching exact. Add hostile-input regressions for the CodeQL-reported readiness flows without changing flagged production paths, destinations or authentication. Specific false-positive dispositions still require approval. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/commands/sre.test.ts | 13 +- controller/src/sre_authority.rs | 110 +++++ controller/src/sre_registration.rs | 6 +- .../templates/crd-karssreregistration.yaml | 24 ++ .../2026-09-08-sre-authority-prerequisite.md | 46 +++ inference-router/src/sre_proxy/tests.rs | 2 + .../src/sre_proxy/tests/request_boundary.rs | 376 ++++++++++++++++++ tests/e2e/sre_authority/credential_paths.py | 33 +- tests/e2e/sre_authority/harness_test.py | 21 +- 9 files changed, 619 insertions(+), 12 deletions(-) create mode 100644 inference-router/src/sre_proxy/tests/request_boundary.rs diff --git a/cli/src/commands/sre.test.ts b/cli/src/commands/sre.test.ts index 1a7f98f48..0bd6fa586 100644 --- a/cli/src/commands/sre.test.ts +++ b/cli/src/commands/sre.test.ts @@ -18,10 +18,13 @@ const controller = JSON.stringify({ function authority(args: readonly string[]): { stdout: string } { const metadata = (name: string, uid = name) => ({ name, uid, resourceVersion: "1", generation: 1 }); + const getIndex = args.indexOf("get"); + const resource = getIndex < 0 ? undefined : args[getIndex + 1]; + const name = getIndex < 0 ? undefined : args[getIndex + 2]; let value: unknown; if (args.includes("can-i")) return { stdout: "yes" }; - if (args.includes("crd")) value = { metadata: metadata("karssreregistrations.kars.azure.com") }; - if (args.includes("karssreregistrations.kars.azure.com")) value = { + if (resource === "crd") value = { metadata: metadata("karssreregistrations.kars.azure.com") }; + if (resource === "karssreregistrations.kars.azure.com") value = { metadata: metadata("canonical"), spec: { enabled: true, @@ -32,7 +35,7 @@ function authority(args: readonly string[]): { stdout: string } { }, status: { phase: "Ready", observedGeneration: 1, privacyRevision: "kars.azure.com/sre-privacy/v2" }, }; - if (args.includes("namespace")) value = args.includes("kars-sre") + if (resource === "namespace") value = name === "kars-sre" ? { metadata: { ...metadata("kars-sre"), annotations: { "kars.azure.com/namespace-claim-version": "v1", "kars.azure.com/sandbox-namespace": "kars-system", @@ -40,11 +43,11 @@ function authority(args: readonly string[]): { stdout: string } { "kars.azure.com/sandbox-uid": "source", } } } : { metadata: metadata("kars-system") }; - if (args.includes("karssandbox")) value = { + if (resource === "karssandbox") value = { metadata: { ...metadata("sre", "source"), namespace: "kars-system", annotations: { "kars.azure.com/namespace-uid": "kars-sre" } }, }; - if (args.includes("clusterrolebindings") || args.includes("rolebindings")) value = { items: [] }; + if (resource === "clusterrolebindings" || resource === "rolebindings") value = { items: [] }; return { stdout: value ? JSON.stringify(value) : "" }; } diff --git a/controller/src/sre_authority.rs b/controller/src/sre_authority.rs index 69c53eaf3..738b826f4 100644 --- a/controller/src/sre_authority.rs +++ b/controller/src/sre_authority.rs @@ -16,6 +16,8 @@ mod privacy_tests; mod tests; use crate::sre_registration::{KarsSRERegistration, NAME, RegistrationStatus}; +use crate::status::conditions; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; use kube::{ Api, Client, ResourceExt, api::{Patch, PatchParams}, @@ -23,6 +25,53 @@ use kube::{ pub(crate) use live::{api_error, check_secret_denial, privacy_epoch}; +fn registration_conditions( + prior: &[Condition], + generation: Option, + phase: &str, + detail: Option<&str>, +) -> Vec { + let (reason, message) = match phase { + "Ready" => ("AuthorityReady", "Reviewed SRE authority is ready"), + "Retired" => ("AuthorityRetired", "Private SRE authority has been retired"), + "Provisioning" => ("Provisioning", "Private SRE identity is being provisioned"), + "Migrating" => ( + "Migrating", + "Reviewed SRE grants and consumers are migrating", + ), + _ => ("AuthorityBlocked", "SRE authority cannot be established"), + }; + let mut result = prior.to_vec(); + for (kind, active) in [ + (conditions::TYPE_READY, phase == "Ready"), + ( + conditions::TYPE_PROGRESSING, + matches!(phase, "Migrating" | "Provisioning"), + ), + ( + conditions::TYPE_DEGRADED, + !matches!(phase, "Ready" | "Retired" | "Migrating" | "Provisioning"), + ), + ] { + conditions::set( + &mut result, + conditions::preserve_transition_time( + conditions::find(prior, kind), + kind, + if active { + conditions::status::TRUE + } else { + conditions::status::FALSE + }, + reason, + detail.unwrap_or(message), + generation, + ), + ); + } + result +} + async fn status( client: &Client, reg: &KarsSRERegistration, @@ -34,6 +83,15 @@ async fn status( let status = RegistrationStatus { phase: phase.into(), observed_generation: reg.metadata.generation.unwrap_or_default(), + conditions: registration_conditions( + reg.status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or_default(), + reg.metadata.generation, + phase, + detail.as_deref(), + ), privacy_epoch: (phase == "Ready").then(|| reg.epoch()), router_service_account_uid: if phase == "Retired" { None @@ -187,3 +245,55 @@ pub async fn run(client: Client) { tokio::time::sleep(std::time::Duration::from_secs(20)).await; } } + +#[cfg(test)] +mod status_tests { + use super::*; + + #[test] + fn registration_phases_publish_truthful_standard_conditions() { + for (phase, expected) in [ + ("Ready", ["True", "False", "False"]), + ("Retired", ["False", "False", "False"]), + ("Migrating", ["False", "True", "False"]), + ("Provisioning", ["False", "True", "False"]), + ("Blocked", ["False", "False", "True"]), + ("UnknownPhase", ["False", "False", "True"]), + ] { + let result = registration_conditions(&[], Some(3), phase, None); + assert_eq!(result.len(), 3); + for (kind, expected) in ["Ready", "Progressing", "Degraded"] + .into_iter() + .zip(expected) + { + let condition = conditions::find(&result, kind).unwrap(); + assert_eq!(condition.status, expected, "{phase}/{kind}"); + assert_eq!(condition.observed_generation, Some(3)); + assert!(!condition.reason.is_empty()); + assert!(!condition.message.is_empty()); + } + } + } + + #[test] + fn repeated_status_preserves_transition_time_and_unrelated_conditions() { + let mut prior = registration_conditions(&[], Some(1), "Migrating", None); + conditions::set( + &mut prior, + conditions::new_condition("CustomEvidence", "True", "Observed", "preserve", Some(1)), + ); + let next = registration_conditions(&prior, Some(2), "Migrating", Some("Still draining")); + assert_eq!(next.len(), 4); + assert_eq!( + conditions::find(&next, "CustomEvidence"), + conditions::find(&prior, "CustomEvidence") + ); + for kind in ["Ready", "Progressing", "Degraded"] { + let before = conditions::find(&prior, kind).unwrap(); + let after = conditions::find(&next, kind).unwrap(); + assert_eq!(before.last_transition_time, after.last_transition_time); + assert_eq!(after.observed_generation, Some(2)); + assert_eq!(after.message, "Still draining"); + } + } +} diff --git a/controller/src/sre_registration.rs b/controller/src/sre_registration.rs index 93e7861b3..13b552b2f 100644 --- a/controller/src/sre_registration.rs +++ b/controller/src/sre_registration.rs @@ -5,6 +5,7 @@ //! Namespace ownership identifies an occupant; only this resource delegates //! privileged SRE authority to that exact occupant. +use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -69,7 +70,8 @@ pub struct ConsumerReview { kind = "KarsSRERegistration", plural = "karssreregistrations", status = "RegistrationStatus", - printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"# + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"# )] #[serde(rename_all = "camelCase")] pub struct KarsSRERegistrationSpec { @@ -93,6 +95,8 @@ fn enabled() -> bool { pub struct RegistrationStatus { pub phase: String, pub observed_generation: i64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conditions: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub privacy_epoch: Option, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/deploy/helm/kars/templates/crd-karssreregistration.yaml b/deploy/helm/kars/templates/crd-karssreregistration.yaml index cc9374408..5553b2af6 100644 --- a/deploy/helm/kars/templates/crd-karssreregistration.yaml +++ b/deploy/helm/kars/templates/crd-karssreregistration.yaml @@ -2,6 +2,9 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: karssreregistrations.kars.azure.com + labels: + app.kubernetes.io/name: kars + app.kubernetes.io/component: crd annotations: helm.sh/resource-policy: keep spec: @@ -15,6 +18,13 @@ spec: - name: v1alpha1 served: true storage: true + additionalPrinterColumns: + - name: Phase + type: string + jsonPath: .status.phase + - name: Age + type: date + jsonPath: .metadata.creationTimestamp subresources: status: {} schema: @@ -106,6 +116,20 @@ spec: status: type: object properties: + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [type, status, reason, message, lastTransitionTime] + properties: + type: {type: string} + status: {type: string, enum: ["True", "False", "Unknown"]} + reason: {type: string} + message: {type: string} + observedGeneration: {type: integer, format: int64, minimum: 0} + lastTransitionTime: {type: string, format: date-time} phase: {type: string} observedGeneration: {type: integer, format: int64} privacyEpoch: {type: string} diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 94d649a8f..3c26226fd 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -143,3 +143,49 @@ remains strict. The shared Rust privacy helper is included in capability-audit, no-stub, no-custom-crypto and runtime-affecting Kind path classification. Its location outside the individual crates is not a security-gate exception. + +## Full-CI follow-up + +The initial full run exposed omitted registration labels, printer columns and +standard conditions. The repair adds real Ready/Progressing/Degraded condition +updates using the existing transition-time helpers, along with the schema and +display metadata; it does not exempt this CRD from conformance. The existing +17-criterion conformance suite passes against the corrected schema. + +Secret-type mutations are tested separately from validating admission: +Kubernetes rejects immutable `type` changes with a specific 422 cause before +the VAP runs. Schema-valid CREATE and annotation updates still require the +intended policy-specific 403. Neither arbitrary errors nor HTTP 200 watches +count as denial. + +Four Rust CodeQL alerts are being investigated without suppression or product +rewriting. Their reported flows start at the readiness handler's injected Axum +State. Current evidence identifies fixed credential filenames under the +production mount and a controller-configured Kubernetes origin, rather than an +HTTP-selected location. Independent boundary review and hostile-input +regressions are pending; no false-positive classification or alert dismissal +has been approved. + +### Confirm-boundary-first evidence + +Independent source review of SARIF analysis `1739818791` recommends classifying +alerts 780/781 (path injection) and 782/783 (request forgery) as false positives +for the reported HTTP-input flows. Each flow starts at `get(ready)` and treats +Axum `State` as request data. Pinned Axum 0.8.9 instead clones the supplied +server state and ignores request parts. The sole production constructor uses +`/etc/kars/sre-api`; filenames are literals, and Kubernetes origin/namespace +come from the controller-generated private configuration. + +Regression-only coverage now sends eight hostile header/query/body scenarios +through the actual readiness handler. It forces projected-file rereads, token +renewal and metadata inventory: each scenario records 24 calls to the selected +Kubernetes endpoint, while alternate HTTP/HTTPS servers receive none. The +alternate credential files are not selected. Startup-constant provenance is +also checked. The flagged production files and lockfile remain unchanged. + +All 50 focused SRE tests and strict combined controller/router Clippy pass, +including the now correctly registered condition tests. This evidence assumes +trusted controller/kubelet configuration and private-volume integrity; it does +not excuse privileged configuration tampering. Approval for the four specific +false-positive dispositions is still pending. No query, source path, or alert +has been suppressed or dismissed. diff --git a/inference-router/src/sre_proxy/tests.rs b/inference-router/src/sre_proxy/tests.rs index 28c257253..a9ab83633 100644 --- a/inference-router/src/sre_proxy/tests.rs +++ b/inference-router/src/sre_proxy/tests.rs @@ -8,6 +8,8 @@ use serde_json::json; use std::sync::Mutex; use wiremock::{Mock, MockServer, ResponseTemplate}; +mod request_boundary; + const PRIVATE_VALUE: &str = "PRIVATE_OPERATOR_CONTROL_VALUE"; struct Fixture { diff --git a/inference-router/src/sre_proxy/tests/request_boundary.rs b/inference-router/src/sre_proxy/tests/request_boundary.rs new file mode 100644 index 000000000..18e41e590 --- /dev/null +++ b/inference-router/src/sre_proxy/tests/request_boundary.rs @@ -0,0 +1,376 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::super::*; +use chrono::{Duration, Utc}; +use serde_json::{Value, json}; +use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +const TOKEN: &str = "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router/token"; +const REGISTRATION: &str = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical"; +const NAMESPACE: &str = "/api/v1/namespaces/kars-sre"; +const SOURCE: &str = "/apis/kars.azure.com/v1alpha1/namespaces/kars-system/karssandboxes/sre"; +const ACCOUNT: &str = "/api/v1/namespaces/kars-sre/serviceaccounts/sre-api-router"; +const REVIEWS: &str = "/apis/authorization.k8s.io/v1/subjectaccessreviews"; +const INVENTORY: &str = "/api/v1/namespaces/kars-sre/secrets"; +const PROJECTED_TOKEN: &str = "selected-projected-token"; +const RENEWED_TOKEN: &str = "selected-renewed-token"; + +#[derive(Clone)] +struct RecordedRequest { + method: Method, + uri: Uri, + headers: HeaderMap, + body: Value, +} + +type Requests = Arc>>; + +async fn kubernetes( + State(requests): State, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let body: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + requests.lock().unwrap().push(RecordedRequest { + method: method.clone(), + uri: uri.clone(), + headers: headers.clone(), + body: body.clone(), + }); + let expected = if uri.path() == TOKEN { + PROJECTED_TOKEN + } else { + RENEWED_TOKEN + }; + if headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + != Some(format!("Bearer {expected}").as_str()) + { + return error(StatusCode::UNAUTHORIZED, "Unexpected test credential"); + } + let value = match (method, uri.path()) { + (Method::POST, TOKEN) => json!({"status":{ + "token":RENEWED_TOKEN, + "expirationTimestamp":(Utc::now()+Duration::hours(1)).to_rfc3339(), + }}), + (Method::GET, REGISTRATION) => json!({ + "metadata":{"uid":"registration","generation":1}, + "spec":{"enabled":true,"sandbox":{"namespace":"kars-system","uid":"source"}, + "runtimeNamespace":{"uid":"namespace"}}, + "status":{"phase":"Ready","observedGeneration":1,"privacyEpoch":"epoch", + "legacySecretAccessDenied":true,"privacyRevision":crate::sre_privacy::REVISION}, + }), + (Method::GET, NAMESPACE) => json!({"metadata":{"uid":"namespace","annotations":{ + "kars.azure.com/namespace-claim-version":"v1", + "kars.azure.com/sandbox-namespace":"kars-system", + "kars.azure.com/sandbox-name":"sre", + "kars.azure.com/sandbox-uid":"source", + }}}), + (Method::GET, SOURCE) => json!({"metadata":{"uid":"source", + "annotations":{"kars.azure.com/namespace-uid":"namespace"}}}), + (Method::GET, ACCOUNT) => json!({"metadata":{"uid":"router-sa"}}), + (Method::POST, REVIEWS) => json!({"status":{"allowed":false}}), + (Method::GET, INVENTORY) => json!({ + "apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList", + "metadata":{},"items":[], + }), + _ => return error(StatusCode::NOT_FOUND, "Unexpected test Kubernetes path"), + }; + axum::Json(value).into_response() +} + +async fn attacker(State(hits): State>) -> &'static str { + hits.fetch_add(1, Ordering::SeqCst); + "attacker server reached" +} + +struct Server { + origin: String, + authority: String, + task: tokio::task::JoinHandle<()>, +} + +impl Server { + async fn start(router: Router, identity: Option<&Path>) -> Self { + let tcp = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let address = tcp.local_addr().unwrap(); + let task = if let Some(directory) = identity { + let listener = Listener { + tcp, + tls: tls(directory).unwrap(), + }; + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }) + } else { + tokio::spawn(async move { axum::serve(tcp, router).await.unwrap() }) + }; + Self { + origin: format!( + "{}://{address}", + if identity.is_some() { "https" } else { "http" } + ), + authority: address.to_string(), + task, + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +fn write_identity(directory: &Path) -> String { + let key = rcgen::KeyPair::generate().unwrap(); + let certificate = rcgen::CertificateParams::new(vec!["127.0.0.1".into(), "localhost".into()]) + .unwrap() + .self_signed(&key) + .unwrap(); + let pem = certificate.pem(); + std::fs::write(directory.join("server-cert.pem"), &pem).unwrap(); + std::fs::write(directory.join("server-key.pem"), key.serialize_pem()).unwrap(); + std::fs::write(directory.join("kube-ca.crt"), &pem).unwrap(); + pem +} + +async fn exercise_readiness_inputs(channel: &str, use_https_attacker: bool) { + let directory = tempfile::tempdir_in(".").unwrap(); + let selected = directory.path().join("selected"); + let alternate = directory.path().join("alternate"); + std::fs::create_dir(&selected).unwrap(); + std::fs::create_dir(&alternate).unwrap(); + let ca = write_identity(&selected); + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(reqwest::Certificate::from_pem(ca.as_bytes()).unwrap()) + .timeout(std::time::Duration::from_secs(10)) + .build() + .unwrap(); + let hits = Arc::new(AtomicUsize::new(0)); + let attacker_app = Router::new().fallback(attacker).with_state(hits.clone()); + let http_attacker = Server::start(attacker_app.clone(), None).await; + let https_attacker = Server::start(attacker_app, Some(&selected)).await; + + // Both attacker origins are reachable; the HTTPS attacker is trusted by + // this test CA too, so a bad destination cannot be masked by TLS rejection. + for origin in [&http_attacker.origin, &https_attacker.origin] { + assert_eq!( + client + .get(origin) + .send() + .await + .unwrap() + .text() + .await + .unwrap(), + "attacker server reached" + ); + } + assert_eq!(hits.swap(0, Ordering::SeqCst), 2); + let foreign = if use_https_attacker { + &https_attacker + } else { + &http_attacker + }; + let requests: Requests = Arc::new(Mutex::new(Vec::new())); + let kube = Server::start( + Router::new() + .fallback(kubernetes) + .with_state(requests.clone()), + Some(&selected), + ) + .await; + let configuration = json!({ + "schema":"kars.azure.com/sre-api/v1","kubeUrl":kube.origin, + "registrationUid":"registration","privacyEpoch":"epoch", + "source":{"namespace":"kars-system","name":"sre","uid":"source"}, + "runtimeNamespace":"kars-sre","namespaceUid":"namespace", + "serviceAccountUid":"router-sa","secretUid":"selected-secret", + }); + std::fs::write( + selected.join("config.json"), + serde_json::to_vec(&configuration).unwrap(), + ) + .unwrap(); + std::fs::write(selected.join("kube-token"), "initial-selected-token").unwrap(); + std::fs::write( + selected.join("kube-expires-at"), + (Utc::now() + Duration::minutes(2)).to_rfc3339(), + ) + .unwrap(); + + // Use the production loader and its HTTPS/CA/redirect configuration, not + // Backend::for_test. Only fixture setup selects this project-local volume. + let backend = Backend::load(&selected).unwrap(); + assert!(requests.lock().unwrap().is_empty()); + std::fs::write(selected.join("kube-token"), PROJECTED_TOKEN).unwrap(); + std::fs::write( + selected.join("kube-expires-at"), + (Utc::now() + Duration::minutes(4)).to_rfc3339(), + ) + .unwrap(); + std::fs::write(alternate.join("kube-token"), "attacker-alternate-token").unwrap(); + std::fs::write(alternate.join("kube-expires-at"), "invalid-attacker-expiry").unwrap(); + let mut hostile_configuration = configuration.clone(); + hostile_configuration["kubeUrl"] = foreign.origin.clone().into(); + hostile_configuration["runtimeNamespace"] = "kars-attacker".into(); + std::fs::write( + alternate.join("config.json"), + serde_json::to_vec(&hostile_configuration).unwrap(), + ) + .unwrap(); + + let proxy = Proxy { + backend, + token: Arc::from("a".repeat(64)), + capacity: Arc::new(Semaphore::new(16)), + }; + let server = Server::start(app(proxy), Some(&selected)).await; + let mut request = client.get(format!("{}/readyz", server.origin)); + let alternate_name = alternate.canonicalize().unwrap().display().to_string(); + if matches!(channel, "query" | "all") { + request = request.query(&[ + ("directory", alternate_name.as_str()), + ("credentialDirectory", alternate_name.as_str()), + ("kubeUrl", foreign.origin.as_str()), + ("runtimeNamespace", "kars-attacker"), + ]); + } + if matches!(channel, "headers" | "all") { + request = request + .header("host", &foreign.authority) + .header("x-forwarded-host", &foreign.authority) + .header( + "forwarded", + format!("host={};proto=https", foreign.authority), + ) + .header("origin", &foreign.origin) + .header("x-credential-directory", &alternate_name) + .header("x-kube-url", &foreign.origin) + .header("x-runtime-namespace", "kars-attacker"); + } + if matches!(channel, "body" | "all") { + request = request.json(&json!({ + "directory":alternate_name,"kubeUrl":foreign.origin,"runtimeNamespace":"kars-attacker", + "proxy":{"backend":{"directory":alternate_name,"config":hostile_configuration}}, + })); + } + let response = request.send().await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{channel}"); + assert_eq!( + response.json::().await.unwrap(), + json!({"ready":true}) + ); + assert_eq!(hits.load(Ordering::SeqCst), 0, "{channel}"); + + let calls = requests.lock().unwrap().clone(); + assert_eq!( + calls.len(), + 24, + "{channel}: renewal + four live identities + 18 reviews + inventory" + ); + assert_eq!(calls[0].method, Method::POST); + assert_eq!(calls[0].uri.path(), TOKEN); + assert_eq!( + calls[0].headers["authorization"], + format!("Bearer {PROJECTED_TOKEN}") + ); + assert_eq!( + calls[0].body["spec"]["boundObjectRef"], + json!({"apiVersion":"v1","kind":"Secret","name":"sre-api-router-identity","uid":"selected-secret"}) + ); + for call in &calls { + assert_eq!(call.headers["host"], kube.authority); + assert!(call.uri.query().is_none()); + assert!( + [ + TOKEN, + REGISTRATION, + NAMESPACE, + SOURCE, + ACCOUNT, + REVIEWS, + INVENTORY + ] + .contains(&call.uri.path()) + ); + assert!(!call.headers.contains_key("x-credential-directory")); + assert!(!call.headers.contains_key("x-kube-url")); + assert!(!call.headers.contains_key("x-forwarded-host")); + assert!( + !serde_json::to_string(&call.body) + .unwrap() + .contains("kars-attacker") + ); + } + for call in &calls[1..] { + assert_eq!( + call.headers["authorization"], + format!("Bearer {RENEWED_TOKEN}") + ); + } + assert_eq!( + calls.iter().filter(|call| call.uri.path() == TOKEN).count(), + 1 + ); + assert_eq!( + calls + .iter() + .filter(|call| call.uri.path() == REVIEWS) + .count(), + 18 + ); + let inventory = calls + .iter() + .find(|call| call.uri.path() == INVENTORY) + .unwrap(); + assert_eq!(inventory.method, Method::GET); + assert!( + inventory.headers["accept"] + .to_str() + .unwrap() + .contains("PartialObjectMetadataList") + ); + assert_eq!( + std::fs::read_to_string(alternate.join("kube-token")).unwrap(), + "attacker-alternate-token" + ); + assert_eq!( + std::fs::read_to_string(alternate.join("kube-expires-at")).unwrap(), + "invalid-attacker-expiry" + ); +} + +#[tokio::test] +async fn ready_hostile_inputs_cannot_select_files_origin_or_namespace() { + for channel in ["query", "headers", "body", "all"] { + for https in [false, true] { + exercise_readiness_inputs(channel, https).await; + } + } +} + +#[test] +fn production_factory_remains_startup_only_with_a_fixed_credential_directory() { + assert_eq!(DIRECTORY, "/etc/kars/sre-api"); + let production = include_str!("../mod.rs"); + assert_eq!(production.matches("Backend::load(").count(), 1); + let start = production.find("pub async fn start()").unwrap(); + let probe = production.find("pub async fn readiness_probe()").unwrap(); + let startup = &production[start..probe]; + assert!(startup.contains("let directory = PathBuf::from(DIRECTORY);")); + assert!(startup.contains("Backend::load(&directory)")); + let ready_start = production.find("async fn ready(").unwrap(); + let forward_start = production.find("async fn forward(").unwrap(); + let ready = &production[ready_start..forward_start]; + assert!(ready.contains("async fn ready(State(proxy): State)")); + assert!(!ready.contains("Backend::load(")); + assert!(include_str!("../../main.rs").contains("kars_inference_router::sre_proxy::start()")); +} diff --git a/tests/e2e/sre_authority/credential_paths.py b/tests/e2e/sre_authority/credential_paths.py index d30155f39..4f9be1e69 100644 --- a/tests/e2e/sre_authority/credential_paths.py +++ b/tests/e2e/sre_authority/credential_paths.py @@ -16,6 +16,18 @@ SA_NAME = "kubernetes.io/service-account.name" WATCH_MARKER = "kind-dummy-secret-watch-proof" +def assert_token_type_immutable(response, label): + """Secret type validation precedes VAP; this is not admission-policy proof.""" + require(response.status_code == 422, + f"{label}: expected immutable Secret type rejection, got HTTP {response.status_code}") + body = response.json() + require(body.get("kind") == "Status" and body.get("reason") == "Invalid" + and any(cause.get("field") == "type" + and cause.get("reason") == "FieldValueInvalid" + and "field is immutable" in cause.get("message", "") + for cause in body.get("details", {}).get("causes", [])), + f"{label}: missing the specific immutable type validation cause") + def seed_privacy_gaps(h): require(h.get("serviceaccount", "sre-api-router", RUNTIME) is None, @@ -75,21 +87,32 @@ def token_secret_denials(h, before_enrollment=False): "metadata": {"name": f"{prefix}-{suffix}", "namespace": RUNTIME, "annotations": annotations}}, user="tenant", status=201).json() fixtures.append(obj) - assert_denial(h.api("PATCH", path + "/" + obj["metadata"]["name"], body={ + response = h.api("PATCH", path + "/" + obj["metadata"]["name"], body={ **patch, "metadata": {**patch.get("metadata", {}), "uid": obj["metadata"]["uid"], "resourceVersion": obj["metadata"]["resourceVersion"]}}, - user="tenant"), f"token Secret {suffix} PATCH", policy) + user="tenant") + if "type" in patch: + assert_token_type_immutable(response, f"token Secret {suffix} PATCH") + else: + assert_denial(response, f"token Secret {suffix} PATCH", policy) current = h.get("secret", obj["metadata"]["name"], RUNTIME) require(current["metadata"]["resourceVersion"] == obj["metadata"]["resourceVersion"] and not current.get("data"), "Denied token Secret update mutated or populated its fixture") if before_enrollment: for patch in ({"type": "Opaque"}, {"metadata": {"annotations": {SA_NAME: "e2e-other"}}}): - assert_denial(h.api("PATCH", path + "/" + TOKEN_ALIAS, body=patch, user="tenant"), - "prestaged oldObject type/annotation escape", policy) + before = h.get("secret", TOKEN_ALIAS, RUNTIME) + response = h.api("PATCH", path + "/" + TOKEN_ALIAS, body=patch, user="tenant") + if "type" in patch: + assert_token_type_immutable(response, "prestaged oldObject type escape") + else: + assert_denial(response, "prestaged oldObject annotation escape", policy) + require(h.get("secret", TOKEN_ALIAS, RUNTIME) == before, + "Rejected oldObject token escape changed its prestaged fixture") finally: for obj in fixtures: delete_owned(h, path + "/" + obj["metadata"]["name"], obj) - h.passed("Tenant and registrar arbitrary token Secret CREATE, plus type/annotation PATCH escapes, receive exact admission denials") + h.passed("Tenant and registrar token Secret CREATE and schema-valid annotation updates receive exact admission denials") + h.passed("Token-type mutations receive specific Kubernetes immutable-field errors without changing fixtures") def fixture_review(h): diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index c6324cd13..f77102ac3 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -17,7 +17,7 @@ Harness, assert_claim, assert_denial, enrollment_json, printed_object, review_args, ) from sre_authority.proxy import MARKER, assert_filtered -from sre_authority.credential_paths import assert_watch_result +from sre_authority.credential_paths import assert_token_type_immutable, assert_watch_result class Response: @@ -30,6 +30,25 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_secret_type_rejection_is_specific_and_not_counted_as_vap_denial(self): + cause = {"field": "type", "reason": "FieldValueInvalid", + "message": 'Invalid value: "Opaque": field is immutable'} + body = {"kind": "Status", "reason": "Invalid", "details": {"causes": [cause]}} + assert_token_type_immutable(Response(422, body), "type update") + with self.assertRaises(AssertionError): + assert_denial(Response(422, body), "type update", "kars-sre-no-legacy-tokens") + for code in (200, 400, 403, 404, 409, 500): + with self.subTest(code=code), self.assertRaises(AssertionError): + assert_token_type_immutable(Response(code, body), "type update") + for invalid in ( + {**body, "reason": "Forbidden"}, + {**body, "details": {"causes": []}}, + {**body, "details": {"causes": [{**cause, "field": "metadata.annotations"}]}}, + {**body, "details": {"causes": [{**cause, "message": "different validation failure"}]}}, + ): + with self.subTest(body=invalid), self.assertRaises(AssertionError): + assert_token_type_immutable(Response(422, invalid), "type update") + def test_denials_require_real_forbidden_and_the_intended_policy(self): valid = {"kind": "Status", "reason": "Forbidden", "message": "kars-sre-private-mounts denied"} assert_denial(Response(403, valid), "probe", "kars-sre-private-mounts") From 997876825aa54cded225b2ef4a82050b2e5bc8c8 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 13:07:23 +0200 Subject: [PATCH 04/62] test(sre): initialize TLS provider in isolated boundary regression Select the existing AWS-LC provider before this test creates TLS clients or servers, so nextest isolation does not rely on another test initializing process state. Record the user-approved dispositions for the four specific CodeQL HTTP-flow false positives; production proxy paths and destinations remain unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../2026-09-08-sre-authority-prerequisite.md | 22 +++++++++++++------ .../src/sre_proxy/tests/request_boundary.rs | 1 + 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 3c26226fd..009b35ac1 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -158,13 +158,12 @@ the VAP runs. Schema-valid CREATE and annotation updates still require the intended policy-specific 403. Neither arbitrary errors nor HTTP 200 watches count as denial. -Four Rust CodeQL alerts are being investigated without suppression or product +Four Rust CodeQL alerts were investigated without suppression or product rewriting. Their reported flows start at the readiness handler's injected Axum State. Current evidence identifies fixed credential filenames under the production mount and a controller-configured Kubernetes origin, rather than an -HTTP-selected location. Independent boundary review and hostile-input -regressions are pending; no false-positive classification or alert dismissal -has been approved. +HTTP-selected location. The independent assessment and the approved +per-alert dispositions are recorded below. ### Confirm-boundary-first evidence @@ -186,6 +185,15 @@ also checked. The flagged production files and lockfile remain unchanged. All 50 focused SRE tests and strict combined controller/router Clippy pass, including the now correctly registered condition tests. This evidence assumes trusted controller/kubelet configuration and private-volume integrity; it does -not excuse privileged configuration tampering. Approval for the four specific -false-positive dispositions is still pending. No query, source path, or alert -has been suppressed or dismissed. +not excuse privileged configuration tampering. The user explicitly approved +false-positive dispositions for only alerts 780, 781, 782 and 783; each GitHub +alert now carries its specific evidence comment. No query or source path was +excluded, and no other alert was dismissed. This is not author/reviewer audit +sign-off, PR approval, or permission to deploy. + +The first full CI execution ran the new boundary case in an isolated process +and exposed missing test-local Rustls provider initialization. Other tests had +initialized it during the earlier grouped run. The case now explicitly selects +the existing AWS-LC provider before any TLS setup and passes when executed +alone. Production proxy code and the reported input boundaries remain unchanged; +hosted Kubernetes migration qualification is still required. diff --git a/inference-router/src/sre_proxy/tests/request_boundary.rs b/inference-router/src/sre_proxy/tests/request_boundary.rs index 18e41e590..831c89d7a 100644 --- a/inference-router/src/sre_proxy/tests/request_boundary.rs +++ b/inference-router/src/sre_proxy/tests/request_boundary.rs @@ -350,6 +350,7 @@ async fn exercise_readiness_inputs(channel: &str, use_https_attacker: bool) { #[tokio::test] async fn ready_hostile_inputs_cannot_select_files_origin_or_namespace() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); for channel in ["query", "headers", "body", "all"] { for https in [false, true] { exercise_readiness_inputs(channel, https).await; From 3c9523e8508ccd09113806677a8cc60d6e53b6fd Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 15:30:00 +0200 Subject: [PATCH 05/62] test(sre): repair schema-valid authority acceptance fixtures Supply the required inference reference in both legacy control-consumer and reserved-source admission fixtures so real API tests reach their intended policy boundary. Report only harness source coordinates, exit code and an allowlisted category when commands fail; keep credentials and response bodies out of diagnostics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/admission.py | 13 +++++--- tests/e2e/sre_authority/common.py | 40 ++++++++++++++++++++++-- tests/e2e/sre_authority/fixtures.py | 1 + tests/e2e/sre_authority/harness_test.py | 41 +++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/tests/e2e/sre_authority/admission.py b/tests/e2e/sre_authority/admission.py index 9cd2e9cbd..8db2d3b4e 100644 --- a/tests/e2e/sre_authority/admission.py +++ b/tests/e2e/sre_authority/admission.py @@ -37,6 +37,14 @@ def pod_spec(private=True): return spec +def reserved_source_probe(): + return {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": "sre", "namespace": TENANT}, + "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1"}}, + "inferenceRef": {"name": "sre-inference"}, + "sandbox": {"isolation": "standard"}}} + + def admission_cases(h, enrollment): registration = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", "metadata": {"name": "canonical"}, "spec": enrollment} @@ -53,10 +61,7 @@ def admission_cases(h, enrollment): require(review["status"]["allowed"] is False, "Ordinary SA unexpectedly has registrar use") h.passed("Real RBAC and CEL independently deny ordinary/tenant registration and registrar use") - source = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", - "metadata": {"name": "sre", "namespace": TENANT}, - "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1"}}, - "sandbox": {"isolation": "standard"}}} + source = reserved_source_probe() assert_denial(h.api("POST", f"/apis/kars.azure.com/v1alpha1/namespaces/{TENANT}/karssandboxes?dryRun=All", body=source, user="tenant"), "reserved source", "kars-sre-source-authority") source["metadata"]["name"] = "not-canonical" diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 8ca156e72..3d2999bdb 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -4,9 +4,11 @@ import base64 import contextlib import hashlib +import inspect import json import os from pathlib import Path +import re import socket import signal import ssl @@ -44,6 +46,38 @@ def require(condition, message): raise AssertionError(message) +def command_site(): + # Only source coordinates from our harness, never argv, frame locals, + # absolute filesystem paths or a traceback containing API response data. + frame = inspect.currentframe() + try: + while frame: + source = Path(frame.f_code.co_filename) + if source.parent == Path(__file__).parent and source.name != "common.py": + return f"{source.name}:{frame.f_code.co_name}:{frame.f_lineno}" + frame = frame.f_back + return "harness" + finally: + del frame + + +def command_error_category(stderr): + status = re.search(r"Error from server \((Forbidden|Unauthorized|Invalid|NotFound|" + r"AlreadyExists|Conflict|BadRequest|InternalError|ServiceUnavailable)\)", stderr) + if status: + return status.group(1) + text = stderr.lower() + for needle, category in (("error validating", "schema-validation"), + ("unknown flag", "cli-argument"), + ("required value", "required-field"), + ("no matches for kind", "api-discovery"), + ("the server doesn't have a resource type", "api-discovery"), + ("timed out waiting", "wait-timeout")): + if needle in text: + return category + return "unclassified" + + def printed_object(output): start = output.find("{") require(start >= 0, "CLI omitted its JSON object") @@ -173,11 +207,13 @@ def run(self, args, *, data=None, user="admin", timeout=35, expected=0): except subprocess.TimeoutExpired: os.killpg(process.pid, signal.SIGKILL) process.communicate(timeout=5) - raise AssertionError(f"Command {Path(args[0]).name} exceeded its bounded timeout") from None + raise AssertionError(f"Command exceeded its bounded timeout at {command_site()}") from None result = subprocess.CompletedProcess(args, process.returncode, stdout, stderr) if expected is not None: # Never echo command output or argv: token/Secret reads are captured. - require(result.returncode == expected, f"Command {Path(args[0]).name} failed during {self.phase}") + require(result.returncode == expected, + f"Command failed during {self.phase} at {command_site()}; " + f"exit={result.returncode}; category={command_error_category(stderr)}") return result if expected is None else result.stdout def k(self, *args, data=None, user="admin", timeout=35, expected=0): diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py index 30157f8bb..3c3f94783 100644 --- a/tests/e2e/sre_authority/fixtures.py +++ b/tests/e2e/sre_authority/fixtures.py @@ -148,6 +148,7 @@ def seed_control_consumer(h): "metadata": {"name": CONTROL, "namespace": SYSTEM}, "spec": {"runtime": {"kind": "BYO", "byo": {"image": STANDIN, "contractVersion": "v1", "command": ["/bin/sh"], "args": ["-c", "sleep infinity"]}}, + "inferenceRef": {"name": "sre-inference"}, "sandbox": {"isolation": "standard"}}}) namespace_uid = namespace_claim(h, source, CONTROL_NS) secret = h.create({"apiVersion": "v1", "kind": "Secret", diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index f77102ac3..5e8f4e3af 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -8,6 +8,7 @@ from pathlib import Path import re import tempfile +import time import types import unittest from unittest.mock import patch @@ -16,6 +17,8 @@ CLAIM_VERSION, NAMESPACE_UID, RUNTIME, SOURCE_NAME, SOURCE_NS, SOURCE_UID, SYSTEM, Harness, assert_claim, assert_denial, enrollment_json, printed_object, review_args, ) +from sre_authority.fixtures import seed_control_consumer +from sre_authority.admission import reserved_source_probe from sre_authority.proxy import MARKER, assert_filtered from sre_authority.credential_paths import assert_token_type_immutable, assert_watch_result @@ -30,6 +33,44 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_control_consumer_fixture_satisfies_required_sandbox_fields(self): + captured = [] + class Captured(Exception): + pass + def capture(obj, **_kwargs): + captured.append(obj) + raise Captured + with self.assertRaises(Captured): + seed_control_consumer(types.SimpleNamespace(create=capture)) + root = Path(__file__).resolve().parents[3] + schema = (root / "deploy/helm/kars/templates/crd.yaml").read_text() + spec = schema.split(" spec:\n", 1)[1] + required = json.loads(re.search(r"required: (\[[^\n]+\])", spec).group(1)) + for source in (captured[0], reserved_source_probe()): + with self.subTest(name=source["metadata"]["name"]): + self.assertTrue(set(required).issubset(source["spec"])) + self.assertEqual(source["spec"]["inferenceRef"], {"name": "sre-inference"}) + self.assertEqual(source["spec"]["runtime"]["kind"], "BYO") + self.assertNotIn("agent", source["spec"]) + + def test_failed_command_reports_only_source_exit_and_allowlisted_category(self): + root = Path(__file__).resolve().parent + with tempfile.TemporaryDirectory(prefix=".harness-unit-", dir=root) as folder: + harness = Harness.__new__(Harness) + harness.work = harness.root = Path(folder) + harness.deadline = time.monotonic() + 30 + harness.phase = "prepare" + process = types.SimpleNamespace(returncode=1, communicate=lambda **_kwargs: ( + MARKER, f"Error from server (Invalid): {MARKER} secret/header/kubeconfig body")) + with patch("sre_authority.common.subprocess.Popen", return_value=process): + with self.assertRaises(AssertionError) as failure: + harness.run(["kubectl", "--token", MARKER], data=MARKER) + message = str(failure.exception) + self.assertIn("harness_test.py:test_failed_command_reports_only_source_exit_and_allowlisted_category:", message) + self.assertIn("exit=1; category=Invalid", message) + for forbidden in (MARKER, "--token", "secret/header/kubeconfig body", str(harness.root)): + self.assertNotIn(forbidden, message) + def test_secret_type_rejection_is_specific_and_not_counted_as_vap_denial(self): cause = {"field": "type", "reason": "FieldValueInvalid", "message": 'Invalid value: "Opaque": field is immutable'} From e9e1038f9619c08f402ae9b904025437ffeb450b Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 16:28:51 +0200 Subject: [PATCH 06/62] ci(sre): validate public registration schema before image-dependent tests Use the existing pinned Kind/kubectl/Helm tools and identical node configuration for a fast independent API-server dry-run. Preserve only allowlisted 422 causes for the exact public SRE CRD, retain generic command privacy, and classify native kubectl CRD Invalid errors. No production or schema change; root cause remains to be established by hosted API evidence. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 45 +++- tests/e2e/run.sh | 2 + tests/e2e/sre_authority/common.py | 2 + tests/e2e/sre_authority/fixtures.py | 3 +- .../e2e/sre_authority/registration_schema.py | 208 ++++++++++++++++++ .../sre_authority/registration_schema_test.py | 146 ++++++++++++ 6 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/sre_authority/registration_schema.py create mode 100644 tests/e2e/sre_authority/registration_schema_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c05451762..fd61e666f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -396,6 +396,45 @@ jobs: - name: RBAC idempotency gate run: python3 ci/bicep-rbac-idempotency.py + sre-crd-schema: + name: SRE CRD API Schema + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + KUBECONFIG: ${{ github.workspace }}/.e2e-sre-schema-kubeconfig + PYTHONDONTWRITEBYTECODE: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Same pinned tools and unmodified node configuration as e2e-kind. + # Kind v0.24.0 defaults to the observed kindest/node:v1.31.0 image. + - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 + with: + install_only: true + version: v0.24.0 + - uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4 + with: + version: v1.30.5 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Check public-schema diagnostic privacy + run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test + - name: Create the same disposable API server as the real harness + run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" + - name: Validate the SRE CRD against the actual API server + run: python3 tests/e2e/sre_authority/registration_schema.py + - name: Upload only public CRD schema evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 + with: + name: sre-crd-schema-${{ github.run_id }} + path: | + e2e-sre-schema-diag/versions.json + e2e-sre-schema-diag/validation.json + if-no-files-found: warn + retention-days: 7 + - name: Remove disposable schema cluster + if: always() + run: kind delete cluster --name kars-e2e + helm-lint: name: Helm Lint runs-on: ubuntu-latest @@ -708,7 +747,11 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 with: name: e2e-diagnostics-${{ github.run_id }} - path: e2e-diag/ + path: | + e2e-diag/ + e2e-sre-schema-diag/versions.json + e2e-sre-schema-diag/validation.json + e2e-sre-schema-diag/registration-create.json retention-days: 7 bench-regression: diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 81e99b304..e1b1322da 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -3049,6 +3049,8 @@ main() { trap teardown EXIT setup_cluster + # Validate the public cluster API before any Rust images or private fixtures. + PYTHONDONTWRITEBYTECODE=1 python3 "$SCRIPT_DIR/sre_authority/registration_schema.py" build_images prepare_sre_authority_legacy install_crds diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 3d2999bdb..547d75c60 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -66,6 +66,8 @@ def command_error_category(stderr): r"AlreadyExists|Conflict|BadRequest|InternalError|ServiceUnavailable)\)", stderr) if status: return status.group(1) + if re.search(r'The CustomResourceDefinition "[^"\r\n]+" is invalid:', stderr): + return "Invalid" text = stderr.lower() for needle, category in (("error validating", "schema-validation"), ("unknown flag", "cli-argument"), diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py index 3c3f94783..6653a2939 100644 --- a/tests/e2e/sre_authority/fixtures.py +++ b/tests/e2e/sre_authority/fixtures.py @@ -11,6 +11,7 @@ enrollment_json, fingerprint, require, ) from .credential_paths import seed_privacy_gaps +from .registration_schema import create_registration_crd LEGACY_COMMIT = "8b206065608593667a40665b3f48225ef9ce278d" CONTROL = "e2e-control-rotation" @@ -128,7 +129,7 @@ def prepare_legacy(h): obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" obj["metadata"].setdefault("annotations", {}).update({ "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM}) - h.create(obj) + create_registration_crd(h, obj) h.k("wait", "--for=condition=Established", "crd/karssreregistrations.kars.azure.com", "--timeout=60s", timeout=70) before = h.get("clusterrolebinding", "kars-sre-reader") h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", diff --git a/tests/e2e/sre_authority/registration_schema.py b/tests/e2e/sre_authority/registration_schema.py new file mode 100644 index 000000000..640ade784 --- /dev/null +++ b/tests/e2e/sre_authority/registration_schema.py @@ -0,0 +1,208 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Real API validation for one public CRD; never a generic error-body logger.""" + +import contextlib +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, build_opener, ProxyHandler + +CONTEXT = "kind-kars-e2e" +CRD_NAME = "karssreregistrations.kars.azure.com" +CRD_PATH = "/apis/apiextensions.k8s.io/v1/customresourcedefinitions" +TEMPLATE = "templates/crd-karssreregistration.yaml" +REPORT_DIR = "e2e-sre-schema-diag" + + +def require_public_crd(obj): + if (not isinstance(obj, dict) or obj.get("apiVersion") != "apiextensions.k8s.io/v1" + or obj.get("kind") != "CustomResourceDefinition" + or obj.get("metadata", {}).get("name") != CRD_NAME + or obj.get("spec", {}).get("group") != "kars.azure.com" + or obj.get("spec", {}).get("scope") != "Cluster" + or obj.get("spec", {}).get("names", {}).get("kind") != "KarsSRERegistration" + or obj.get("spec", {}).get("names", {}).get("plural") != "karssreregistrations" + or set(obj) - {"apiVersion", "kind", "metadata", "spec"}): + raise AssertionError("Schema diagnostic accepts only the public SRE registration CRD") + + +def public_status(code, body): + """Only this known public schema's Invalid causes may cross the log boundary.""" + report = {"resource": CRD_NAME, "httpStatus": code, "category": "unexpected-response"} + if not isinstance(body, dict): + return report + if code in (200, 201) and body.get("kind") == "CustomResourceDefinition": + if body.get("metadata", {}).get("name") == CRD_NAME: + report["category"] = "accepted" + return report + reasons = {"Invalid", "Forbidden", "Unauthorized", "NotFound", "AlreadyExists", + "Conflict", "BadRequest", "InternalError", "ServiceUnavailable"} + if body.get("kind") == "Status" and body.get("reason") in reasons: + report["category"] = body["reason"] + details = body.get("details", {}) + if (code != 422 or body.get("kind") != "Status" or body.get("reason") != "Invalid" + or not isinstance(details, dict) or details.get("name") != CRD_NAME + or details.get("group") != "apiextensions.k8s.io" + or details.get("kind") != "CustomResourceDefinition"): + return report + causes = [] + supplied = details.get("causes", []) + if not isinstance(supplied, list): + return report + for cause in supplied[:32]: + if not isinstance(cause, dict): + continue + field, message = cause.get("field"), cause.get("message") + if (not isinstance(field, str) or not field.startswith("spec.") + or not isinstance(message, str)): + continue + causes.append({ + "field": re.sub(r"[\x00-\x1f\x7f]", "?", field)[:1024], + "message": re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "?", message)[:16384], + }) + report["causes"] = causes + return report + + +def write_report(root, filename, report): + directory = Path(root) / REPORT_DIR + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + (directory / filename).write_text(json.dumps(report, indent=2) + "\n") + print("SRE-CRD-SCHEMA " + json.dumps(report, sort_keys=True), flush=True) + + +def create_registration_crd(harness, obj): + require_public_crd(obj) + response = harness.api("POST", CRD_PATH, body=obj) + try: + body = response.json() + except (ValueError, TypeError): + body = None + report = public_status(response.status_code, body) + write_report(harness.root, "registration-create.json", report) + if response.status_code != 201 or report["category"] != "accepted": + raise AssertionError( + f"Public SRE registration CRD rejected: HTTP {response.status_code}; " + f"category={report['category']}; see allowlisted schema causes" + ) + return body + + +def command(stage, args, *, root, data=None): + try: + result = subprocess.run(args, cwd=root, input=data, text=True, capture_output=True, + timeout=45, check=False) + except (OSError, subprocess.TimeoutExpired): + raise RuntimeError(f"Public schema preflight {stage} command unavailable/timed out") from None + if result.returncode: + raise RuntimeError(f"Public schema preflight {stage} failed; exit={result.returncode}") from None + return result.stdout + + +def request(port, method, path, obj=None): + body = None if obj is None else json.dumps(obj).encode() + req = Request(f"http://127.0.0.1:{port}{path}", data=body, method=method, + headers={"Content-Type": "application/json", "Accept": "application/json"}) + opener = build_opener(ProxyHandler({})) + try: + response = opener.open(req, timeout=15) + except HTTPError as error: + response = error + with response: + code = response.code + raw = response.read(1024 * 1024) + try: + return code, json.loads(raw) + except (ValueError, TypeError): + return code, None + + +@contextlib.contextmanager +def kind_proxy(root): + # Read only redacted config to verify the exact disposable context/server. + config = json.loads(command("context", ["kubectl", "--context", CONTEXT, "config", "view", + "--minify", "-o", "json"], root=root)) + contexts, clusters = config.get("contexts", []), config.get("clusters", []) + if (len(contexts) != 1 or contexts[0].get("name") != CONTEXT or len(clusters) != 1 + or urlsplit(clusters[0].get("cluster", {}).get("server", "")).hostname + not in ("localhost", "127.0.0.1", "::1")): + raise RuntimeError("Public schema preflight refuses a non-loopback/non-Kind context") + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + process = subprocess.Popen( + ["kubectl", "--context", CONTEXT, "--request-timeout=15s", "proxy", + "--address=127.0.0.1", f"--port={port}"], + cwd=root, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 25 + version = None + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Public schema preflight proxy exited before readiness") + try: + code, observed = request(port, "GET", "/version") + if code == 200 and isinstance(observed, dict) and isinstance(observed.get("gitVersion"), str): + version = {key: observed.get(key) for key in ("major", "minor", "gitVersion")} + break + except (URLError, TimeoutError): + pass + time.sleep(0.2) + if version is None: + raise RuntimeError("Public schema preflight proxy readiness timed out") + yield port, version + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def preflight(root): + rendered = command("render", ["helm", "template", "kars", str(root / "deploy/helm/kars"), + "--namespace", "kars-system", "--show-only", TEMPLATE], root=root) + # Existing kubectl parses YAML; strict client validation remains enabled. + obj = json.loads(command("conversion", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=rendered)) + require_public_crd(obj) + with kind_proxy(root) as (port, version): + write_report(root, "versions.json", {"apiServer": version, "context": CONTEXT}) + code, existing = request(port, "GET", f"{CRD_PATH}/{CRD_NAME}") + if code == 404: + method, path, accepted = "POST", CRD_PATH, 201 + elif code == 200 and public_status(code, existing)["category"] == "accepted": + method, path, accepted = "PUT", f"{CRD_PATH}/{CRD_NAME}", 200 + for key in ("uid", "resourceVersion"): + obj["metadata"][key] = existing["metadata"][key] + else: + raise RuntimeError(f"Public schema preflight could not inspect CRD; HTTP {code}") + code, body = request(port, method, path + "?dryRun=All", obj) + report = public_status(code, body) + write_report(root, "validation.json", report) + if code != accepted or report["category"] != "accepted": + raise RuntimeError(f"Public SRE registration schema rejected; HTTP {code}") + + +if __name__ == "__main__": + os.umask(0o077) + try: + preflight(Path(__file__).resolve().parents[3]) + except Exception as error: + # Deliberately do not expose arbitrary exception text, command output, + # argv, config material, or an unrelated HTTP response. + print(f"SRE-CRD-SCHEMA-FAIL category={type(error).__name__}", flush=True) + raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/registration_schema_test.py b/tests/e2e/sre_authority/registration_schema_test.py new file mode 100644 index 000000000..89a974d46 --- /dev/null +++ b/tests/e2e/sre_authority/registration_schema_test.py @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure checks for the public-CRD diagnostic boundary, not real API evidence.""" + +import contextlib +import copy +import json +from pathlib import Path +import types +import unittest +from unittest.mock import patch + +from sre_authority.common import command_error_category +from sre_authority import registration_schema as schema + +PRIVATE = "DO-NOT-LOG-PRIVATE-AUTH-OR-SECRET-BODY" + + +def crd(): + return { + "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", + "metadata": {"name": schema.CRD_NAME}, + "spec": {"group": "kars.azure.com", "scope": "Cluster", + "names": {"kind": "KarsSRERegistration", "plural": "karssreregistrations"}}, + } + + +def invalid(): + return { + "kind": "Status", "status": "Failure", "reason": "Invalid", "code": 422, + "details": { + "name": schema.CRD_NAME, "group": "apiextensions.k8s.io", "kind": "CustomResourceDefinition", + "causes": [{"field": "spec.versions[0].schema.openAPIV3Schema.properties[spec].x-kubernetes-validations[0].rule", + "message": "public schema compile diagnostic\n ^"}], + }, + } + + +class RegistrationSchemaTests(unittest.TestCase): + def test_native_kubectl_invalid_classification_does_not_echo_body(self): + message = f'The CustomResourceDefinition "{schema.CRD_NAME}" is invalid: {PRIVATE}' + self.assertEqual(command_error_category(message), "Invalid") + self.assertNotIn(PRIVATE, command_error_category(message)) + self.assertEqual(command_error_category(f"Error from server (Forbidden): {PRIVATE}"), "Forbidden") + + def test_only_exact_public_schema_422_causes_are_exposed(self): + report = schema.public_status(422, invalid()) + self.assertEqual(report["category"], "Invalid") + self.assertEqual(len(report["causes"]), 1) + self.assertIn("public schema compile diagnostic", report["causes"][0]["message"]) + for code, kind, name, group in [ + (403, "CustomResourceDefinition", schema.CRD_NAME, "apiextensions.k8s.io"), + (500, "CustomResourceDefinition", schema.CRD_NAME, "apiextensions.k8s.io"), + (422, "Secret", "private-secret", ""), + (422, "CustomResourceDefinition", "other.example.test", "apiextensions.k8s.io"), + (422, "CustomResourceDefinition", schema.CRD_NAME, "other.group"), + ]: + body = invalid() + body["message"] = PRIVATE + body["details"].update({"kind": kind, "name": name, "group": group}) + body["details"]["causes"][0]["message"] = PRIVATE + with self.subTest(code=code, kind=kind): + self.assertNotIn(PRIVATE, json.dumps(schema.public_status(code, body))) + body = invalid() + body["details"]["causes"] = [{"field": "data.token", "message": PRIVATE}] + self.assertNotIn(PRIVATE, json.dumps(schema.public_status(422, body))) + + def test_malformed_responses_do_not_become_success_or_echo_raw_content(self): + for body in (None, [], PRIVATE, {"message": PRIVATE}, + {**invalid(), "details": None}, + {**invalid(), "details": {**invalid()["details"], "causes": None}}): + with self.subTest(body=type(body).__name__): + report = schema.public_status(422, body) + self.assertNotEqual(report["category"], "accepted") + self.assertNotIn(PRIVATE, json.dumps(report)) + + def test_general_secret_commands_cannot_use_the_public_crd_diagnostic(self): + for obj in ( + {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": schema.CRD_NAME}}, + {**crd(), "data": {"token": PRIVATE}}, + {**crd(), "metadata": {"name": "other.example.test"}}, + ): + with self.assertRaises(AssertionError): + schema.require_public_crd(obj) + + def test_actual_create_path_surfaces_only_allowlisted_api_evidence(self): + response = types.SimpleNamespace(status_code=422, json=invalid) + harness = types.SimpleNamespace(root=Path("."), api=lambda *_args, **_kwargs: response) + with patch.object(schema, "write_report") as write: + with self.assertRaisesRegex(AssertionError, "HTTP 422"): + schema.create_registration_crd(harness, crd()) + self.assertEqual(write.call_args.args[1], "registration-create.json") + self.assertEqual(write.call_args.args[2]["causes"], schema.public_status(422, invalid())["causes"]) + + def test_fast_preflight_uses_real_server_dry_run_and_rejects_invalid_schema(self): + root = Path(__file__).resolve().parents[3] + seen = [] + def request(_port, method, path, body=None): + seen.append((method, path, body)) + return (404, {}) if method == "GET" else (422, invalid()) + def command(stage, _args, **_kwargs): + return "public yaml" if stage == "render" else json.dumps(crd()) + with patch.object(schema, "kind_proxy", return_value=contextlib.nullcontext((1, {"gitVersion": "v1.31.0"}))), \ + patch.object(schema, "request", side_effect=request), \ + patch.object(schema, "command", side_effect=command), \ + patch.object(schema, "write_report") as write: + with self.assertRaisesRegex(RuntimeError, "HTTP 422"): + schema.preflight(root) + self.assertEqual(seen[1][:2], ("POST", schema.CRD_PATH + "?dryRun=All")) + self.assertEqual(write.call_args.args[2]["category"], "Invalid") + + def test_reused_kind_cluster_still_validates_update_instead_of_accepting_already_exists(self): + root = Path(__file__).resolve().parents[3] + existing = crd() + existing["metadata"].update({"uid": "crd-uid", "resourceVersion": "42"}) + seen = [] + def request(_port, method, path, body=None): + seen.append((method, path, copy.deepcopy(body))) + return (200, existing) + with patch.object(schema, "kind_proxy", return_value=contextlib.nullcontext((1, {"gitVersion": "v1.31.0"}))), \ + patch.object(schema, "request", side_effect=request), \ + patch.object(schema, "command", side_effect=lambda stage, *_args, **_kwargs: + "public yaml" if stage == "render" else json.dumps(crd())), \ + patch.object(schema, "write_report"): + schema.preflight(root) + self.assertEqual(seen[1][:2], ("PUT", schema.CRD_PATH + "/" + schema.CRD_NAME + "?dryRun=All")) + self.assertEqual(seen[1][2]["metadata"]["uid"], "crd-uid") + self.assertEqual(seen[1][2]["metadata"]["resourceVersion"], "42") + + def test_fast_ci_gate_uses_the_harness_pins_without_waiting_for_rust_images(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + job = workflow.split(" sre-crd-schema:\n", 1)[1].split(" helm-lint:\n", 1)[0] + for required in ("version: v0.24.0", "version: v1.30.5", "tests/e2e/kind-config.yaml", + "registration_schema.py", "e2e-sre-schema-diag/validation.json"): + self.assertIn(required, job) + for forbidden in ("needs:", "cargo ", "docker/build-push", "--validate=false", "continue-on-error"): + self.assertNotIn(forbidden, job) + runner = (root / "tests/e2e/run.sh").read_text() + self.assertLess(runner.index('python3 "$SCRIPT_DIR/sre_authority/registration_schema.py"'), + runner.index("\n build_images\n")) + + +if __name__ == "__main__": + unittest.main() From e2aefef61d2c74d35a39e62da819cb8759e1c227 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 16:37:09 +0200 Subject: [PATCH 07/62] test(sre): compare API-evidenced CEL namespace accessors Hosted Kubernetes v1.31.0 rejected the public CRD with HTTP 422: both custom namespace field accesses are undefined. Add a diagnostic-only escaped-accessor candidate and real positive/negative instance checks in the disposable schema job. The unchanged production schema must still pass its own gate; candidate success cannot turn that failure green. Production/schema files remain unchanged pending parent review. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 8 +- .../e2e/sre_authority/registration_schema.py | 83 ++++++++++++++++++- .../sre_authority/registration_schema_test.py | 40 +++++++++ 3 files changed, 127 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd61e666f..2c3ffc78a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -420,7 +420,10 @@ jobs: - name: Create the same disposable API server as the real harness run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" - name: Validate the SRE CRD against the actual API server - run: python3 tests/e2e/sre_authority/registration_schema.py + run: python3 tests/e2e/sre_authority/registration_schema.py --exercise + - name: Collect namespace-accessor candidate evidence without relaxing the failing production gate + if: failure() + run: python3 tests/e2e/sre_authority/registration_schema.py --namespace-accessor-candidate --exercise - name: Upload only public CRD schema evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 @@ -429,6 +432,9 @@ jobs: path: | e2e-sre-schema-diag/versions.json e2e-sre-schema-diag/validation.json + e2e-sre-schema-diag/validation-instances.json + e2e-sre-schema-diag/namespace-accessor-candidate.json + e2e-sre-schema-diag/namespace-accessor-candidate-instances.json if-no-files-found: warn retention-days: 7 - name: Remove disposable schema cluster diff --git a/tests/e2e/sre_authority/registration_schema.py b/tests/e2e/sre_authority/registration_schema.py index 640ade784..2b1f3326b 100644 --- a/tests/e2e/sre_authority/registration_schema.py +++ b/tests/e2e/sre_authority/registration_schema.py @@ -4,6 +4,8 @@ """Real API validation for one public CRD; never a generic error-body logger.""" import contextlib +import argparse +import copy import json import os from pathlib import Path @@ -20,6 +22,8 @@ CRD_PATH = "/apis/apiextensions.k8s.io/v1/customresourcedefinitions" TEMPLATE = "templates/crd-karssreregistration.yaml" REPORT_DIR = "e2e-sre-schema-diag" +ORIGINAL_RULE = "self.sandbox.namespace == self.controller.namespace.name" +ESCAPED_RULE = "self.sandbox.__namespace__ == self.controller.__namespace__.name" def require_public_crd(obj): @@ -170,7 +174,69 @@ def kind_proxy(root): process.wait(timeout=5) -def preflight(root): +def escaped_namespace_candidate(obj): + """Diagnostic-only candidate: preserve the constraint and its wire schema.""" + candidate = copy.deepcopy(obj) + rules = candidate["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["x-kubernetes-validations"] + matches = [rule for rule in rules if rule.get("rule") == ORIGINAL_RULE] + if len(matches) != 1: + raise RuntimeError("Expected public namespace equality rule is absent; candidate probe refused") + matches[0]["rule"] = ESCAPED_RULE + return candidate + + +def exercise_instances(root, port, obj, method, path, accepted, prefix): + # This helper is used only in the isolated schema CI job. The full harness + # keeps its preflight dry-run-only so historical fixture setup is unchanged. + code, body = request(port, method, path, obj) + if code != accepted or public_status(code, body)["category"] != "accepted": + write_report(root, f"{prefix}-install.json", public_status(code, body)) + raise RuntimeError("Public schema could not be installed in the disposable schema cluster") + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + code, current = request(port, "GET", f"{CRD_PATH}/{CRD_NAME}") + if code == 200 and any(condition.get("type") == "Established" and condition.get("status") == "True" + for condition in current.get("status", {}).get("conditions", [])): + break + time.sleep(0.5) + else: + raise RuntimeError("Public registration CRD did not become Established") + instance = { + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, + "spec": { + "controller": {"namespace": {"name": "kars-system", "uid": "fixture-system"}, + "deployment": {"name": "kars-controller", "uid": "fixture-controller"}, "release": "kars"}, + "sandbox": {"namespace": "kars-system", "name": "sre", "uid": "fixture-source"}, + "runtimeNamespace": {"name": "kars-sre", "uid": "fixture-runtime"}, + }, + } + results = [] + for label, expected, message in [ + ("canonical-matching-namespace", 201, None), + ("foreign-source-namespace", 422, "SRE must be registered in its controller/release namespace"), + ("noncanonical-name", 422, "The canonical SRE registration is the only supported instance"), + ]: + probe = copy.deepcopy(instance) + if label == "foreign-source-namespace": + probe["spec"]["sandbox"]["namespace"] = "other-system" + elif label == "noncanonical-name": + probe["metadata"]["name"] = "other" + code, response = request(port, "POST", "/apis/kars.azure.com/v1alpha1/karssreregistrations?dryRun=All", probe) + matched = code == expected + if message: + matched = matched and isinstance(response, dict) and response.get("reason") == "Invalid" and any( + message in cause.get("message", "") for cause in response.get("details", {}).get("causes", []) + ) + else: + matched = matched and isinstance(response, dict) and response.get("kind") == "KarsSRERegistration" + results.append({"case": label, "httpStatus": code, "expectedStatus": expected, "matched": bool(matched)}) + write_report(root, f"{prefix}-instances.json", {"cases": results}) + if not matched: + raise RuntimeError("Public registration instance did not satisfy the exact expected schema invariant") + + +def preflight(root, *, candidate=False, exercise=False): rendered = command("render", ["helm", "template", "kars", str(root / "deploy/helm/kars"), "--namespace", "kars-system", "--show-only", TEMPLATE], root=root) # Existing kubectl parses YAML; strict client validation remains enabled. @@ -179,6 +245,9 @@ def preflight(root): "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", ], root=root, data=rendered)) require_public_crd(obj) + if candidate: + obj = escaped_namespace_candidate(obj) + prefix = "namespace-accessor-candidate" if candidate else "validation" with kind_proxy(root) as (port, version): write_report(root, "versions.json", {"apiServer": version, "context": CONTEXT}) code, existing = request(port, "GET", f"{CRD_PATH}/{CRD_NAME}") @@ -192,15 +261,23 @@ def preflight(root): raise RuntimeError(f"Public schema preflight could not inspect CRD; HTTP {code}") code, body = request(port, method, path + "?dryRun=All", obj) report = public_status(code, body) - write_report(root, "validation.json", report) + write_report(root, f"{prefix}.json", report) if code != accepted or report["category"] != "accepted": raise RuntimeError(f"Public SRE registration schema rejected; HTTP {code}") + if exercise: + exercise_instances(root, port, obj, method, path, accepted, prefix) if __name__ == "__main__": os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--namespace-accessor-candidate", action="store_true", + help="Probe only the API-evidenced field-accessor candidate; does not edit production files") + parser.add_argument("--exercise", action="store_true", help="Exercise public instance invariants in the isolated schema job") + args = parser.parse_args() try: - preflight(Path(__file__).resolve().parents[3]) + preflight(Path(__file__).resolve().parents[3], + candidate=args.namespace_accessor_candidate, exercise=args.exercise) except Exception as error: # Deliberately do not expose arbitrary exception text, command output, # argv, config material, or an unrelated HTTP response. diff --git a/tests/e2e/sre_authority/registration_schema_test.py b/tests/e2e/sre_authority/registration_schema_test.py index 89a974d46..a623f2216 100644 --- a/tests/e2e/sre_authority/registration_schema_test.py +++ b/tests/e2e/sre_authority/registration_schema_test.py @@ -128,6 +128,44 @@ def request(_port, method, path, body=None): self.assertEqual(seen[1][2]["metadata"]["uid"], "crd-uid") self.assertEqual(seen[1][2]["metadata"]["resourceVersion"], "42") + def test_candidate_changes_only_field_accessors_not_constraint_or_wire_properties(self): + original = crd() + original["spec"]["versions"] = [{"schema": {"openAPIV3Schema": { + "x-kubernetes-validations": [{"rule": "self.metadata.name == 'canonical'"}], + "properties": {"spec": { + "properties": {"sandbox": {"properties": {"namespace": {"type": "string"}}}}, + "x-kubernetes-validations": [{"rule": schema.ORIGINAL_RULE, "message": "unchanged"}], + }}, + }}}] + expected = copy.deepcopy(original) + expected["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"][ + "x-kubernetes-validations"][0]["rule"] = schema.ESCAPED_RULE + candidate = schema.escaped_namespace_candidate(original) + self.assertEqual(candidate, expected) + self.assertIn(schema.ORIGINAL_RULE, json.dumps(original)) + + def test_candidate_instance_checks_require_both_original_invariants(self): + seen = [] + def request(_port, method, path, body=None): + seen.append((method, path, body)) + if path == schema.CRD_PATH: + return 201, crd() + if method == "GET": + return 200, {**crd(), "status": {"conditions": [{"type": "Established", "status": "True"}]}} + message = None + if body["metadata"]["name"] != "canonical": + message = "The canonical SRE registration is the only supported instance" + elif body["spec"]["sandbox"]["namespace"] != body["spec"]["controller"]["namespace"]["name"]: + message = "SRE must be registered in its controller/release namespace" + if message: + return 422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [{"message": message}]}} + return 201, body + with patch.object(schema, "request", side_effect=request), patch.object(schema, "write_report") as write: + schema.exercise_instances(Path("."), 1, crd(), "POST", schema.CRD_PATH, 201, "candidate") + self.assertEqual(len(write.call_args.args[2]["cases"]), 3) + self.assertTrue(all(case["matched"] for case in write.call_args.args[2]["cases"])) + self.assertTrue(all("?dryRun=All" in path for method, path, _ in seen if "karssreregistrations?" in path)) + def test_fast_ci_gate_uses_the_harness_pins_without_waiting_for_rust_images(self): root = Path(__file__).resolve().parents[3] workflow = (root / ".github/workflows/ci.yml").read_text() @@ -137,6 +175,8 @@ def test_fast_ci_gate_uses_the_harness_pins_without_waiting_for_rust_images(self self.assertIn(required, job) for forbidden in ("needs:", "cargo ", "docker/build-push", "--validate=false", "continue-on-error"): self.assertNotIn(forbidden, job) + self.assertIn("--namespace-accessor-candidate --exercise", job) + self.assertIn("if: failure()", job) runner = (root / "tests/e2e/run.sh").read_text() self.assertLess(runner.index('python3 "$SCRIPT_DIR/sre_authority/registration_schema.py"'), runner.index("\n build_images\n")) From 447312d1afb0a77ccbd2d144d74a4a293a6ceedd Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 16:54:05 +0200 Subject: [PATCH 08/62] fix(sre): escape custom namespace fields in registration CEL Kubernetes v1.31 rejects the custom namespace accesses as undefined fields. Use the schema's CEL-escaped accessors while keeping the wire fields, namespace equality and singleton checks unchanged. Disposable API evidence accepts the corrected CRD and canonical instance and rejects foreign-namespace and noncanonical instances. Full SRE migration qualification remains required. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/testing/sre-authority.test.ts | 14 ++++++++++++++ .../kars/templates/crd-karssreregistration.yaml | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cli/src/testing/sre-authority.test.ts b/cli/src/testing/sre-authority.test.ts index 6599b6de4..1dd59be25 100644 --- a/cli/src/testing/sre-authority.test.ts +++ b/cli/src/testing/sre-authority.test.ts @@ -11,6 +11,20 @@ const root=fileURLToPath(new URL("../../../",import.meta.url)); const chart=fileURLToPath(new URL("../../../deploy/helm/kars",import.meta.url)); describe("SRE authority chart and mutation integration",()=>{ + it("escapes custom namespace fields in CEL without renaming the wire fields or weakening equality",()=>{ + const output=execFileSync("helm",["template","kars",chart,"--show-only","templates/crd-karssreregistration.yaml"],{encoding:"utf8"}); + const registration=parseAllDocuments(output).map(doc=>doc.toJSON()).find(Boolean); + const schema=registration.spec.versions[0].schema.openAPIV3Schema; + const spec=schema.properties.spec; + expect(spec.properties.sandbox.properties.namespace.type).toBe("string"); + expect(spec.properties.controller.properties.namespace.properties.name.type).toBe("string"); + expect(spec["x-kubernetes-validations"]).toEqual([{ + rule:"self.sandbox.__namespace__ == self.controller.__namespace__.name", + message:"SRE must be registered in its controller/release namespace", + }]); + expect(schema["x-kubernetes-validations"][0].rule).toBe("self.metadata.name == 'canonical'"); + }); + it("creates a cluster registration and no default registrar or runtime privilege bindings",()=>{ for(const enabled of [false,true]){ const output=execFileSync("helm",["template","kars",chart,"--namespace","kars-system","--set",`sre.enabled=${enabled}`],{encoding:"utf8"}); diff --git a/deploy/helm/kars/templates/crd-karssreregistration.yaml b/deploy/helm/kars/templates/crd-karssreregistration.yaml index 5553b2af6..1f4b1add8 100644 --- a/deploy/helm/kars/templates/crd-karssreregistration.yaml +++ b/deploy/helm/kars/templates/crd-karssreregistration.yaml @@ -111,7 +111,7 @@ spec: name: {type: string} namespace: {type: string} x-kubernetes-validations: - - rule: "self.sandbox.namespace == self.controller.namespace.name" + - rule: "self.sandbox.__namespace__ == self.controller.__namespace__.name" message: "SRE must be registered in its controller/release namespace" status: type: object From 86162297a2ee6f6044c9487b3a978a8c2fe47a46 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 17:56:35 +0200 Subject: [PATCH 09/62] fix(cli): preserve SRE release discovery on Helm 3 and 4 Helm 4.2.4 in real Kind acceptance rejects the removed list --all flag before staging. Retain Helm 3 behavior; retry only its exact unsupported-flag error after confirming Helm 4, whose default inventory includes all statuses. Share this bounded compatibility path across SRE install and authority stage. Propagate all other discovery errors and preserve context, existing releases, ownership checks and staging semantics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/commands/sre.test.ts | 22 +++++++++ cli/src/commands/sre.ts | 7 +-- cli/src/lib/sre-authority.test.ts | 32 ++++++++++++ cli/src/lib/sre-helm.test.ts | 82 +++++++++++++++++++++++++++++++ cli/src/lib/sre-helm.ts | 26 ++++++++++ cli/src/lib/sre-stage.ts | 3 +- docs/how-to/sre-authority.md | 6 +++ 7 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 cli/src/lib/sre-helm.test.ts create mode 100644 cli/src/lib/sre-helm.ts diff --git a/cli/src/commands/sre.test.ts b/cli/src/commands/sre.test.ts index 0bd6fa586..ab72ec438 100644 --- a/cli/src/commands/sre.test.ts +++ b/cli/src/commands/sre.test.ts @@ -23,6 +23,7 @@ function authority(args: readonly string[]): { stdout: string } { const name = getIndex < 0 ? undefined : args[getIndex + 2]; let value: unknown; if (args.includes("can-i")) return { stdout: "yes" }; + if (resource === "deployment" && name === "kars-controller") value = JSON.parse(controller); if (resource === "crd") value = { metadata: metadata("karssreregistrations.kars.azure.com") }; if (resource === "karssreregistrations.kars.azure.com") value = { metadata: metadata("canonical"), @@ -61,6 +62,27 @@ beforeEach(() => { afterEach(() => vi.restoreAllMocks()); describe("SRE controller upgrade namespace preflight", () => { + it("retains the existing release when Helm 4 removes the all-status flag", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + execute.mockImplementation(async (file, args) => { + if (file === "helm" && args[0] === "list") { + if (args.includes("--all")) { + throw Object.assign(new Error("Unsupported flag"), { + exitCode: 1, stderr: "Error: unknown flag: --all\n", + }); + } + expect(args).toContain("--kube-context"); + return { stdout: releases }; + } + if (file === "helm" && args[0] === "version") return { stdout: "v4.2.4" }; + if (file === "kubectl" && args.includes("karssandboxes")) return { stdout: '{"items":[]}' }; + return authority(args); + }); + await sreCommand().parseAsync(["node", "sre", "install", "--no-wait", "--context", "test-context"]); + expect(execute.mock.calls.some(([file, args]) => file === "helm" && args[0] === "upgrade")).toBe(true); + expect(execute.mock.calls.some(([file, args]) => file === "helm" && args[0] === "install")).toBe(false); + }); + it.each(["upgrade", "template"])("preflights %s mode in the selected context before mutations", async mode => { execute.mockImplementation(async (file, args) => { if (file === "helm" && args[0] === "list") return { stdout: mode === "upgrade" ? releases : "[]" }; diff --git a/cli/src/commands/sre.ts b/cli/src/commands/sre.ts index 6ecc47d80..166d31321 100644 --- a/cli/src/commands/sre.ts +++ b/cli/src/commands/sre.ts @@ -9,6 +9,7 @@ import { inspectNamespaceOwnership } from "../lib/namespace-ownership.js"; import { authorityCommand } from "./sre-authority.js"; import { assertDestroySafe, assertSafeMutation, enroll, get, preview, registration, requireRegistrar, waitForAuthority } from "../lib/sre-authority.js"; import { stageSource } from "../lib/sre-source.js"; +import { listSreHelmReleases } from "../lib/sre-helm.js"; const HELM_RELEASE_NAME = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; @@ -82,9 +83,9 @@ export function sreCommand(): Command { // C. no chart at all → install unprivileged core first, then // atomically create and enroll the fresh SRE source. let mode: "upgrade" | "template" | "install" = "install"; - const listArgs = ["list", "-n", options.namespace, "--all", "-o", "json"]; - if (options.context) listArgs.push("--kube-context", options.context); - const { stdout: releasesOutput } = await execa("helm", listArgs, { stdio: "pipe", timeout: 30_000 }); + const releasesOutput = await listSreHelmReleases((file, args, commandOptions) => + execa(file, [...args, ...(options.context ? ["--kube-context", options.context] : [])], + { ...commandOptions, timeout: 30_000 }), options.namespace); const releases: unknown = JSON.parse(releasesOutput); if (!Array.isArray(releases) || releases.some(release => !release || typeof release !== "object" || typeof release.name !== "string" diff --git a/cli/src/lib/sre-authority.test.ts b/cli/src/lib/sre-authority.test.ts index 404a56ea2..9b34835bd 100644 --- a/cli/src/lib/sre-authority.test.ts +++ b/cli/src/lib/sre-authority.test.ts @@ -196,6 +196,38 @@ describe("SRE cluster registrar boundary",()=>{ } finally { vi.useRealTimers(); } }); + it.each([ + {version:"v3.16.4",dryRun:true}, {version:"v3.16.4",dryRun:false}, + {version:"v4.2.4",dryRun:true}, {version:"v4.2.4",dryRun:false}, + ])("stages an existing release with $version (dry-run: $dryRun)",async({version,dryRun})=>{ + const warn=vi.spyOn(console,"warn").mockImplementation(()=>{}); + try { + const f=fixture(); + f.objects["deployment/kars-system/kars-controller"].spec={ + template:{spec:{serviceAccountName:"kars-controller"}}, + }; + const execute=vi.fn(async(file,args,options)=>{ + if(file==="helm"&&args[0]==="list") { + if(version.startsWith("v4.")&&args.includes("--all")) { + throw Object.assign(new Error("Unsupported flag"),{ + exitCode:1,stderr:"Error: unknown flag: --all\n", + }); + } + return {stdout:'[{"name":"kars","namespace":"kars-system","status":"pending-upgrade"}]'}; + } + if(file==="helm"&&args[0]==="version")return {stdout:version}; + if(file==="helm"&&args[0]==="upgrade")return {stdout:""}; + return f.execute(file,args,options); + }); + await stageAuthority(execute,"chart","kars-system","kars","new/controller:latest","new/router:latest",dryRun); + const upgrade=execute.mock.calls.find(([file,args])=>file==="helm"&&args[0]==="upgrade"); + expect(upgrade?.[1]).toContain("--reset-then-reuse-values"); + expect(upgrade?.[1]).toContain("sre.authorityStage=true"); + expect(upgrade?.[1]).toContain(dryRun?"--dry-run=server":"--wait"); + expect(execute.mock.calls.some(([,args])=>["install","template","create","patch"].includes(args[0]))).toBe(false); + } finally { warn.mockRestore(); } + }); + it("stages legacy template installations using owned resources and controller CAS without overwriting other env",async()=>{ const f=fixture(); const controller=f.objects["deployment/kars-system/kars-controller"]; diff --git a/cli/src/lib/sre-helm.test.ts b/cli/src/lib/sre-helm.test.ts new file mode 100644 index 000000000..b3bc8f513 --- /dev/null +++ b/cli/src/lib/sre-helm.test.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { execa } from "execa"; +import type { Execute } from "./sre-authority.js"; +import { listSreHelmReleases } from "./sre-helm.js"; + +const inventory = JSON.stringify([{ name: "kars", namespace: "workspace", status: "pending-upgrade" }]); +const unsupported = () => Object.assign(new Error("Helm flag rejected"), { + exitCode: 1, stderr: "Error: unknown flag: --all\n", +}); + +afterEach(() => vi.restoreAllMocks()); + +describe("SRE Helm release inventory compatibility", () => { + it("handles the installed Helm flag parser without contacting a cluster", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + // Help keeps listing offline, but still rejects unsupported flags. + const execute: Execute = (file, args, options) => + execa(file, [...args, ...(args[0] === "list" ? ["--help"] : [])], options); + const output = await listSreHelmReleases(execute, "workspace"); + expect(output).toContain("helm list"); + expect(output).toContain("--all-namespaces"); + }); + + it("retains Helm 3 all-status discovery without an extra probe", async () => { + const execute = vi.fn().mockResolvedValue({ stdout: inventory }); + expect(await listSreHelmReleases(execute, "workspace")).toBe(inventory); + expect(execute.mock.calls).toEqual([ + ["helm", ["list", "-n", "workspace", "--all", "-o", "json"], { stdio: "pipe" }], + ]); + }); + + it.each(["v4.0.0", "v4.2.4", "v4.1.0-rc.1+build.2"])( + "uses confirmed Helm %s all-status defaults and preserves executor context", async version => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const execute = vi.fn() + .mockRejectedValueOnce(unsupported()) + .mockResolvedValueOnce({ stdout: version }) + .mockResolvedValueOnce({ stdout: inventory }); + const contextual: Execute = (file, args, options) => + execute(file, ["--kube-context", "test-context", ...args], options); + expect(await listSreHelmReleases(contextual, "workspace")).toBe(inventory); + expect(execute.mock.calls.map(([, args]) => args)).toEqual([ + ["--kube-context", "test-context", "list", "-n", "workspace", "--all", "-o", "json"], + ["--kube-context", "test-context", "version", "--template", "{{.Version}}"], + ["--kube-context", "test-context", "list", "-n", "workspace", "-o", "json"], + ]); + }, + ); + + it.each([ + { exitCode: 1, stderr: "Forbidden: release storage access denied" }, + { exitCode: 1, stderr: "Error: unknown flag: --all-namespaces" }, + { exitCode: 1, stderr: "Error: unknown flag: --all\nanother failure" }, + { exitCode: 2, stderr: "Error: unknown flag: --all" }, + ])("does not retry another discovery failure: %j", async detail => { + const error = Object.assign(new Error("Discovery failed"), detail); + const execute = vi.fn().mockRejectedValue(error); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toBe(error); + expect(execute).toHaveBeenCalledTimes(1); + }); + + it.each(["v3.19.0", "v5.0.0", "", "unexpected output"])( + "does not assume default all-status semantics for %j", async version => { + const execute = vi.fn().mockRejectedValueOnce(unsupported()) + .mockResolvedValueOnce({ stdout: version }); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toThrow("only Helm 4"); + expect(execute).toHaveBeenCalledTimes(2); + }, + ); + + it("propagates a failed Helm 4 retry rather than reporting an absent release", async () => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + const denied = new Error("Forbidden"); + const execute = vi.fn().mockRejectedValueOnce(unsupported()) + .mockResolvedValueOnce({ stdout: "v4.2.4" }).mockRejectedValueOnce(denied); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toBe(denied); + expect(execute).toHaveBeenCalledTimes(3); + }); +}); diff --git a/cli/src/lib/sre-helm.ts b/cli/src/lib/sre-helm.ts new file mode 100644 index 000000000..e885a73fe --- /dev/null +++ b/cli/src/lib/sre-helm.ts @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Execute } from "./sre-authority.js"; + +function removedAllFlag(error: unknown): boolean { + return error instanceof Error + && "exitCode" in error && error.exitCode === 1 + && "stderr" in error && typeof error.stderr === "string" + && error.stderr.trim() === "Error: unknown flag: --all"; +} + +export async function listSreHelmReleases(execute: Execute, namespace: string): Promise { + try { + return (await execute("helm", ["list", "-n", namespace, "--all", "-o", "json"], { stdio: "pipe" })).stdout; + } catch (error) { + if (!removedAllFlag(error)) throw error; + const { stdout } = await execute("helm", ["version", "--template", "{{.Version}}"], { stdio: "pipe" }); + if (!/^v4\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?$/.test(stdout.trim())) { + throw new Error("Unsupported Helm release inventory: only Helm 4 can omit the --all flag.", { cause: error }); + } + // Helm 3 needs --all; Helm 4 removed it and lists every status by default. + console.warn("Helm 4 lists all release statuses by default; retrying without the removed --all flag."); + return (await execute("helm", ["list", "-n", namespace, "-o", "json"], { stdio: "pipe" })).stdout; + } +} diff --git a/cli/src/lib/sre-stage.ts b/cli/src/lib/sre-stage.ts index 30871afa9..e1e77063d 100644 --- a/cli/src/lib/sre-stage.ts +++ b/cli/src/lib/sre-stage.ts @@ -3,6 +3,7 @@ import { parseAllDocuments } from "yaml"; import { get, requireRegistrar, type ApiObject, type Execute } from "./sre-authority.js"; +import { listSreHelmReleases } from "./sre-helm.js"; function parts(image: string): [string,string] { const index=image.lastIndexOf(":"); @@ -26,7 +27,7 @@ export async function stageAuthority( if(controller.spec?.template?.spec?.serviceAccountName!=="kars-controller") { throw new Error("Controller uses a custom ServiceAccount; review and stage its minimal authority role explicitly"); } - const {stdout}=await execute("helm",["list","-n",namespace,"--all","-o","json"],{stdio:"pipe"}); + const stdout=await listSreHelmReleases(execute,namespace); const releases=JSON.parse(stdout) as unknown; if(!Array.isArray(releases)||releases.some(item=>!item||typeof item.name!=="string"||item.namespace!==namespace)) { throw new Error("Helm ownership inventory is invalid"); diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index ed54bd189..e89309d09 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -48,6 +48,12 @@ kars sre authority stage --namespace kars-system --release kars \ Review the output, then run the same command without `--dry-run`. Staging does not enroll an occupant or issue private SRE credentials. +SRE installation and staging inspect all Helm release statuses. Helm 3 uses +`list --all`; Helm 4 removed that flag and lists all statuses by default. The +CLI retries without it only after that exact flag error and a confirmed Helm 4 +version. Other discovery failures remain errors, not an absent release or +permission to install over existing resources. + ```sh kars sre authority preview --namespace kars-system --release kars kars sre authority enroll --namespace kars-system --release kars \ From 7f7a4ac498a0f50c68e2c8762e39f3184c31492a Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 18:41:15 +0200 Subject: [PATCH 10/62] test(sre): defer custom readiness until explicit migration The staged legacy fixtures cannot acknowledge inference policies before enrollment. During this setup only, use Helm 4 legacy built-in readiness waits, retaining Helm 3 behavior. The immediately following real SRE migration gate remains mandatory and fatal before unrelated tests; no production readiness or policy is relaxed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/run.sh | 8 ++++++- tests/e2e/sre-authority.sh | 8 +++++++ tests/e2e/sre_authority/harness_test.py | 32 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index e1b1322da..44c1fa4bc 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -159,6 +159,7 @@ install_crds() { # KARS_E2E_* env vars; the defaults here cover local runs. local replicas="${KARS_E2E_CONTROLLER_REPLICAS:-1}" local disable_le="${KARS_E2E_DISABLE_LEADER_ELECTION:-1}" + local helm_wait_arg=--wait local extra_set_args=( --set "controller.replicas=${replicas}" --set "inferenceRouter.replicas=${replicas}" @@ -185,6 +186,11 @@ install_crds() { # start the qualified controller while retaining the reviewed shapes. extra_set_args+=(--set sre.enabled=true --set sre.authorityStage=true --set-string runtimes.hermes.image=kars-sandbox-e2e:dev) + # Consumer policy acknowledgements cannot become Ready before explicit + # enrollment below. Wait for built-ins here; migration remains fatal. + local helm_version + helm_version=$(helm version --template '{{.Version}}') || return 1 + helm_wait_arg=$(sre_migration_helm_wait_arg "$helm_version") || return 1 fi if ! helm upgrade --install kars "$ROOT_DIR/deploy/helm/kars" \ --namespace kars-system \ @@ -198,7 +204,7 @@ install_crds() { --set sandbox.image.repository=kars-sandbox-e2e \ --set sandbox.image.tag=dev \ "${extra_set_args[@]}" \ - --wait --timeout 5m; then + "$helm_wait_arg" --timeout 5m; then warn "Helm install did not converge within 5m — dumping diagnostics" kubectl get all -n kars-system || true kubectl describe pod -n kars-system -l app.kubernetes.io/component=controller || true diff --git a/tests/e2e/sre-authority.sh b/tests/e2e/sre-authority.sh index fd4376c87..41aa3cac4 100644 --- a/tests/e2e/sre-authority.sh +++ b/tests/e2e/sre-authority.sh @@ -3,6 +3,14 @@ # Licensed under the MIT License. # Called only by the explicit disposable Kind harness. No live/default context. +sre_migration_helm_wait_arg() { + case "$1" in + v3.*) printf '%s\n' --wait ;; + v4.*) printf '%s\n' --wait=legacy ;; + *) printf '%s\n' "Unsupported Helm version for staged SRE acceptance" >&2; return 1 ;; + esac +} + sre_authority_phase() { local phase="$1" output result=0 line info "SRE authority acceptance: ${phase}" diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 5e8f4e3af..926b8f8d5 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -7,6 +7,7 @@ import json from pathlib import Path import re +import subprocess import tempfile import time import types @@ -237,6 +238,37 @@ def test_legacy_seed_precedes_new_install_and_other_acceptance(self): from sre_authority.common import POLICIES self.assertIn("no-legacy-tokens", POLICIES) + def test_staged_setup_waits_for_builtins_without_skipping_migration_readiness(self): + root = Path(__file__).resolve().parents[3] + helper = root / "tests/e2e/sre-authority.sh" + for version, expected in (("v3.16.4", "--wait"), ("v4.2.4", "--wait=legacy")): + with self.subTest(version=version): + result = subprocess.run( + ["bash", "-c", 'source "$1"; sre_migration_helm_wait_arg "$2"', + "sre-wait-test", str(helper), version], + capture_output=True, text=True, check=True, timeout=5, + ) + self.assertEqual(result.stdout.strip(), expected) + for version in ("v5.0.0", "unexpected"): + result = subprocess.run( + ["bash", "-c", 'source "$1"; sre_migration_helm_wait_arg "$2"', + "sre-wait-test", str(helper), version], + capture_output=True, text=True, check=False, timeout=5, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + script = (root / "tests/e2e/run.sh").read_text() + install = script.split("install_crds() {", 1)[1].split("\nteardown()", 1)[0] + self.assertIn("local helm_wait_arg=--wait", install) + staged = install.split('if [ "$SRE_LEGACY_PREPARED" = "1" ]; then', 1)[1].split("\n fi", 1)[0] + self.assertIn('sre_migration_helm_wait_arg "$helm_version"', staged) + self.assertIn('"$helm_wait_arg" --timeout 5m', install) + main = script.split("main() {", 1)[1] + self.assertIn(" install_crds\n", main) + self.assertIn(" test_sre_authority_migration\n", main) + self.assertLess(main.index("install_crds"), main.index("test_sre_authority_migration")) + self.assertLess(main.index("test_sre_authority_migration"), main.index("test_create_sandbox")) + def test_kind_prerequisites_do_not_remove_the_rust_httpx_test_step(self): root = Path(__file__).resolve().parents[3] workflow = (root / ".github/workflows/ci.yml").read_text() From bc392ce78a859a62709ff8dfd53e06d63d28c135 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 19:46:26 +0200 Subject: [PATCH 11/62] test(sre): capture controller admission before teardown Add a disposable image-free controller Pod admission proof and bounded policy/ReplicaSet diagnostics. Preserve migration/readiness gates and publish no workload specs, credentials, argv or generic API bodies. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 9 +- tests/e2e/run.sh | 4 +- .../sre_authority/bootstrap_diagnostics.py | 129 +++++++++++++ tests/e2e/sre_authority/bootstrap_probe.py | 175 ++++++++++++++++++ .../e2e/sre_authority/bootstrap_probe_test.py | 103 +++++++++++ 5 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/sre_authority/bootstrap_diagnostics.py create mode 100644 tests/e2e/sre_authority/bootstrap_probe.py create mode 100644 tests/e2e/sre_authority/bootstrap_probe_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c3ffc78a..e82f1dd68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,13 +416,16 @@ jobs: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Check public-schema diagnostic privacy - run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test + run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test - name: Create the same disposable API server as the real harness run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" - name: Validate the SRE CRD against the actual API server + id: sre_schema run: python3 tests/e2e/sre_authority/registration_schema.py --exercise + - name: Prove controller Pod admission with all chart policies and no image execution + run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe - name: Collect namespace-accessor candidate evidence without relaxing the failing production gate - if: failure() + if: failure() && steps.sre_schema.outcome == 'failure' run: python3 tests/e2e/sre_authority/registration_schema.py --namespace-accessor-candidate --exercise - name: Upload only public CRD schema evidence if: always() @@ -435,6 +438,7 @@ jobs: e2e-sre-schema-diag/validation-instances.json e2e-sre-schema-diag/namespace-accessor-candidate.json e2e-sre-schema-diag/namespace-accessor-candidate-instances.json + e2e-sre-schema-diag/bootstrap-*.json if-no-files-found: warn retention-days: 7 - name: Remove disposable schema cluster @@ -758,6 +762,7 @@ jobs: e2e-sre-schema-diag/versions.json e2e-sre-schema-diag/validation.json e2e-sre-schema-diag/registration-create.json + e2e-sre-schema-diag/bootstrap-*.json retention-days: 7 bench-regression: diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 44c1fa4bc..00b97055b 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -206,9 +206,9 @@ install_crds() { "${extra_set_args[@]}" \ "$helm_wait_arg" --timeout 5m; then warn "Helm install did not converge within 5m — dumping diagnostics" + PYTHONPATH="$ROOT_DIR/tests/e2e" python3 -m sre_authority.bootstrap_probe --diagnostics-only \ + || warn "Bounded controller admission diagnostics were incomplete" kubectl get all -n kars-system || true - kubectl describe pod -n kars-system -l app.kubernetes.io/component=controller || true - kubectl logs -n kars-system -l app.kubernetes.io/component=controller --tail=200 || true return 1 fi } diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py new file mode 100644 index 000000000..7eb75dce7 --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -0,0 +1,129 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Allowlisted public admission evidence; never dump Pod specs, argv or bodies.""" + +import re + +REASONS = { + "Forbidden", "Invalid", "InternalError", "BadRequest", "NotFound", "AlreadyExists", + "Unauthorized", "Conflict", "ServiceUnavailable", "FailedCreate", "ReplicaFailure", + "MinimumReplicasUnavailable", "MinimumReplicasAvailable", "NewReplicaSetCreated", + "FoundNewReplicaSet", "ReplicaSetUpdated", "NewReplicaSetAvailable", + "ReplicaSetCreateError", "ProgressDeadlineExceeded", "FailedScheduling", + "FailedMount", "Failed", "SuccessfulCreate", "ScalingReplicaSet", +} +FIELDS = { + "namespace", "name", "labels", "annotations", "spec", "status", "subjects", + "enabled", "serviceAccountName", "containers", "initContainers", "ephemeralContainers", + "securityContext", "operation", "subResource", "container", "kind", "apiVersion", +} + + +def identifier(value): + return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None + + +def failure_facts(message, policies): + if not isinstance(message, str): + return {} + message = message[:65536] + facts = { + "policies": sorted(name for name in policies if name in message), + "validationMessages": [], + "categories": [label for label, needle in [ + ("compilation", "compilation"), ("evaluation", "evaluation"), + ("no-such-key", "no such key"), ("undefined-field", "undefined field"), + ("undeclared-reference", "undeclared reference"), ("forbidden", "forbidden"), + ("not-found", "not found"), ("quota", "quota"), ("type-checking", "type checking"), + ] if needle in message.lower()], + "missingFields": sorted(set(re.findall(r"no such key: ([A-Za-z_][A-Za-z0-9_]*)", message)) & FIELDS), + } + for name, policy in policies.items(): + for index, validation in enumerate(policy.get("spec", {}).get("validations", [])): + known = validation.get("message") + if isinstance(known, str) and known in message: + facts["validationMessages"].append({"policy": name, "index": index, "message": known}) + facts["serviceAccountMissing"] = 'serviceaccount "kars-controller" not found' in message.lower() + return facts + + +def api_result(code, body, policies): + report = {"httpStatus": code} + if isinstance(body, dict) and body.get("kind") == "Status": + reason = body.get("reason") + report["reason"] = reason if reason in REASONS else "unclassified" + report.update(failure_facts(body.get("message"), policies)) + return report + + +def object_status(obj, policies): + metadata, status = obj.get("metadata", {}), obj.get("status", {}) + report = {"kind": obj.get("kind"), "name": identifier(metadata.get("name")), + "namespace": identifier(metadata.get("namespace")), "uid": identifier(metadata.get("uid"))} + for field in ("replicas", "readyReplicas", "availableReplicas", "observedGeneration"): + if isinstance(status.get(field), int): + report[field] = status[field] + conditions = [] + for condition in status.get("conditions", [])[:16]: + if not isinstance(condition, dict): + continue + entry = {"type": identifier(condition.get("type")), "status": condition.get("status") + if condition.get("status") in ("True", "False", "Unknown") else None, + "reason": condition.get("reason") if condition.get("reason") in REASONS else "unclassified"} + entry.update(failure_facts(condition.get("message"), policies)) + conditions.append(entry) + report["conditions"] = conditions + return report + + +def policy_status(obj, expected): + name, status = obj.get("metadata", {}).get("name"), obj.get("status", {}) + if name not in expected: + raise AssertionError("Only rendered public policies may publish type-check diagnostics") + warnings = [] + for warning in status.get("typeChecking", {}).get("expressionWarnings", [])[:32]: + field, text = warning.get("fieldRef"), warning.get("warning") + if (isinstance(field, str) and re.fullmatch( + r"spec\.(matchConditions|validations|variables)\[\d+\]\.expression", field) + and isinstance(text, str)): + warnings.append({"fieldRef": field, "warning": re.sub( + r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "?", text)[:16384]}) + return {"name": name, "generation": obj.get("metadata", {}).get("generation"), + "observedGeneration": status.get("observedGeneration"), + "typeChecked": "typeChecking" in status, "warnings": warnings} + + +def collect(port, policies, request): + report = {"scope": "controller-Pod-creation-only", "policies": [], "workloads": [], "events": []} + for name in sorted(policies): + code, obj = request(port, "GET", f"/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/{name}") + report["policies"].append(policy_status(obj, policies) if code == 200 else {"name": name, "httpStatus": code}) + owners = set() + for group, plural in [("apis/apps/v1", "deployments"), ("apis/apps/v1", "replicasets"), + ("api/v1", "pods")]: + code, body = request(port, "GET", f"/{group}/namespaces/kars-system/{plural}") + if code != 200 or not isinstance(body, dict): + report["workloads"].append({"kind": plural, "httpStatus": code}) + continue + for obj in body.get("items", [])[:256]: + metadata = obj.get("metadata", {}) + if not identifier(metadata.get("uid")): + continue + if metadata.get("name") == "kars-controller" or any( + owner.get("uid") in owners for owner in metadata.get("ownerReferences", [])): + owners.add(metadata.get("uid")) + obj = dict(obj, kind={"deployments": "Deployment", "replicasets": "ReplicaSet", "pods": "Pod"}[plural]) + report["workloads"].append(object_status(obj, policies)) + code, body = request(port, "GET", "/api/v1/namespaces/kars-system/events?fieldSelector=type%3DWarning") + if code == 200 and isinstance(body, dict): + for event in body.get("items", [])[:256]: + involved = event.get("involvedObject", {}) + if involved.get("uid") not in owners or event.get("reason") not in REASONS: + continue + entry = {"kind": involved.get("kind") if involved.get("kind") in ("Deployment", "ReplicaSet", "Pod") else None, + "name": identifier(involved.get("name")), + "reason": event.get("reason"), "count": event.get("count")} + entry.update(failure_facts(event.get("message"), policies)) + report["events"].append(entry) + return report diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py new file mode 100644 index 000000000..ffd8a62b7 --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -0,0 +1,175 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Actual disposable API Pod-creation proof without executing or pulling images.""" + +import argparse +import copy +import json +import os +from pathlib import Path +import re +import time + +from sre_authority.bootstrap_diagnostics import api_result, collect +from sre_authority.registration_schema import CONTEXT, command, kind_proxy, request, write_report + +PATHS = { + "CustomResourceDefinition": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions", + "ServiceAccount": "/api/v1/namespaces/{namespace}/serviceaccounts", + "ClusterRole": "/apis/rbac.authorization.k8s.io/v1/clusterroles", + "ClusterRoleBinding": "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings", + "Role": "/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/roles", + "RoleBinding": "/apis/rbac.authorization.k8s.io/v1/namespaces/{namespace}/rolebindings", + "ValidatingAdmissionPolicy": "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies", + "ValidatingAdmissionPolicyBinding": "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings", + "Deployment": "/apis/apps/v1/namespaces/{namespace}/deployments", +} + + +def builtin_documents(rendered): + documents = [] + for document in re.split(r"(?m)^---\s*\n", rendered): + kinds = re.findall(r"(?m)^kind:\s*([A-Za-z]+)\s*$", document) + if len(kinds) == 1 and kinds[0] in PATHS: + documents.append(document) + return "\n---\n".join(documents) + + +def chart(root): + rendered = command("bootstrap-render", [ + "helm", "template", "kars", str(root / "deploy/helm/kars"), "--namespace", "kars-system", + # Offline rendering cannot satisfy the deliberate live-source Helm lookup. + # The disabled-core stage emits the same controller and admission policies, + # without creating or executing an SRE source. + "--kube-version", "1.31.0", "--set", "sre.enabled=false", "--set", "sre.authorityStage=true", + "--set", "controller.replicas=1", "--set", "inferenceRouter.replicas=1", + "--set-string", "inferenceRouter.azure.openai.endpoint=https://e2e-fake.invalid/", + "--set-string", "foundry.endpoint=https://e2e-fake.invalid/", + "--set-string", "foundry.projectEndpoint=https://e2e-fake.invalid/", + ], root=root) + converted = json.loads(command("bootstrap-conversion", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=builtin_documents(rendered))) + objects = converted.get("items", []) if converted.get("kind") == "List" else [converted] + if not objects or any(obj.get("kind") not in PATHS for obj in objects): + raise RuntimeError("Public bootstrap chart contained unexpected converted objects") + return objects + + +def safe_controller(obj): + if obj.get("kind") != "Deployment" or obj.get("metadata", {}).get("name") != "kars-controller": + raise AssertionError("Only the public controller template is a bootstrap fixture") + fixture = copy.deepcopy(obj) + fixture["metadata"]["namespace"] = "kars-system" + fixture["spec"]["replicas"] = 1 + pod = fixture["spec"]["template"]["spec"] + # Admission proof only: no node scheduling, image pulls or container execution. + pod["schedulerName"] = "kars-e2e-admission-never-schedule" + for container in pod.get("containers", []) + pod.get("initContainers", []): + container["image"] = "registry.invalid/kars-admission-proof:never" + container["imagePullPolicy"] = "Never" + return fixture + + +def upsert(port, obj, policies): + metadata = obj.get("metadata", {}) + path = PATHS[obj["kind"]].format(namespace=metadata.get("namespace", "kars-system")) + code, existing = request(port, "GET", f"{path}/{metadata['name']}") + method, accepted = ("PUT", 200) if code == 200 else ("POST", 201) + if code not in (200, 404): + return api_result(code, existing, policies) + desired = copy.deepcopy(obj) + if method == "PUT": + desired["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"] + path += "/" + metadata["name"] + code, body = request(port, method, path, desired) + result = api_result(code, body, policies) + result["accepted"] = code == accepted + result["kind"], result["name"] = obj["kind"], metadata["name"] + return result + + +def exercise(root, port, objects, policies): + results = [] + for name in ("kars-system", "kars-sre"): + code, body = request(port, "POST", "/api/v1/namespaces", { + "apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}}) + if code not in (201, 409): + raise RuntimeError("Disposable bootstrap namespace unavailable") + for kind in PATHS: + if kind == "Deployment": + continue + for obj in objects: + if obj["kind"] != kind: + continue + result = upsert(port, obj, policies) + results.append(result) + if not result.get("accepted"): + write_report(root, "bootstrap-install.json", {"operations": results}) + raise RuntimeError("Disposable public bootstrap admission install failed") + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + snapshot = collect(port, policies, request) + if all(entry.get("typeChecked") and entry.get("generation") == entry.get("observedGeneration") + for entry in snapshot["policies"]): + break + time.sleep(1) + else: + write_report(root, "bootstrap-before-create.json", snapshot) + raise RuntimeError("Public admission policy observation timed out") + write_report(root, "bootstrap-before-create.json", snapshot) + templates = [obj for obj in objects if obj["kind"] == "Deployment" + and obj["metadata"]["name"] == "kars-controller"] + if len(templates) != 1: + raise RuntimeError("Expected one public controller Deployment") + fixture = safe_controller(templates[0]) + template = fixture["spec"]["template"] + pod = {"apiVersion": "v1", "kind": "Pod", + "metadata": {"name": "kars-controller-admission-direct", "namespace": "kars-system", + "labels": template["metadata"]["labels"]}, "spec": template["spec"]} + code, body = request(port, "POST", "/api/v1/namespaces/kars-system/pods?dryRun=All", pod) + direct = api_result(code, body, policies) + result = upsert(port, fixture, policies) + write_report(root, "bootstrap-create.json", {"directPodDryRun": direct, "deployment": result}) + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + snapshot = collect(port, policies, request) + if any(obj.get("kind") == "Pod" for obj in snapshot["workloads"]): + break + time.sleep(1) + write_report(root, "bootstrap-after-create.json", snapshot) + if code != 201 or not result.get("accepted") or not any(obj.get("kind") == "Pod" for obj in snapshot["workloads"]): + raise RuntimeError("Real controller Deployment/ReplicaSet did not create an admission-only Pod") + write_report(root, "bootstrap-result.json", {"podCreation": "accepted", + "workloadExecution": "not-attempted", "readiness": "not-claimed"}) + + +def main(root, diagnostics_only): + with kind_proxy(root) as (port, version): + objects = chart(root) + policies = {obj["metadata"]["name"]: obj for obj in objects + if obj["kind"] == "ValidatingAdmissionPolicy"} + if not policies: + raise RuntimeError("Public admission policies were not rendered") + write_report(root, "bootstrap-versions.json", {"apiServer": version, "context": CONTEXT}) + if diagnostics_only: + write_report(root, "bootstrap-install-failure.json", collect(port, policies, request)) + else: + try: + exercise(root, port, objects, policies) + finally: + write_report(root, "bootstrap-final.json", collect(port, policies, request)) + + +if __name__ == "__main__": + os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--diagnostics-only", action="store_true") + args = parser.parse_args() + try: + main(Path(__file__).resolve().parents[3], args.diagnostics_only) + except Exception as error: + print(f"SRE-BOOTSTRAP-FAIL category={type(error).__name__}", flush=True) + raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py new file mode 100644 index 000000000..a0df704f9 --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +from pathlib import Path +import unittest + +from .bootstrap_diagnostics import api_result, collect, failure_facts, object_status, policy_status +from .bootstrap_probe import builtin_documents, safe_controller + +POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ + {"message": "Private SRE material requires authority"}]}}} + + +class BootstrapProofTests(unittest.TestCase): + def test_failure_metadata_keeps_public_cause_not_body_or_credentials(self): + message = ('Error creating Pod: kars-sre-private-mounts evaluation failed: no such key: namespace; ' + 'token=do-not-publish argv=do-not-publish') + result = object_status({ + "kind": "ReplicaSet", "metadata": {"name": "kars-controller-abc", "uid": "rs-uid", + "annotations": {"credential": "do-not-publish"}}, + "spec": {"template": {"spec": {"containers": [{"env": ["do-not-publish"]}]}}}, + "status": {"replicas": 0, "conditions": [{"type": "ReplicaFailure", "status": "True", + "reason": "FailedCreate", "message": message}]}, + }, POLICIES) + self.assertNotIn("do-not-publish", json.dumps(result)) + self.assertEqual(result["conditions"][0]["reason"], "FailedCreate") + self.assertEqual(result["conditions"][0]["policies"], ["kars-sre-private-mounts"]) + self.assertEqual(result["conditions"][0]["missingFields"], ["namespace"]) + self.assertNotIn("do-not-publish", json.dumps(api_result( + 403, {"kind": "Status", "reason": "Forbidden", "message": message, "details": "do-not-publish"}, POLICIES))) + + def test_type_warning_scope_is_known_public_policy_field_only(self): + obj = {"metadata": {"name": "kars-sre-private-mounts", "generation": 3}, + "status": {"observedGeneration": 3, "typeChecking": {"expressionWarnings": [ + {"fieldRef": "spec.variables[0].expression", "warning": "undefined field namespace"}, + {"fieldRef": "spec.containers[0].env", "warning": "do-not-publish"}, + ]}}} + result = policy_status(obj, POLICIES) + self.assertTrue(result["typeChecked"]) + self.assertEqual(len(result["warnings"]), 1) + self.assertNotIn("do-not-publish", json.dumps(result)) + obj["metadata"]["name"] = "unrelated-policy" + with self.assertRaises(AssertionError): + policy_status(obj, POLICIES) + + def test_no_scheduling_pulls_or_execution_but_original_controller_shape_remains(self): + original = {"kind": "Deployment", "metadata": {"name": "kars-controller"}, + "spec": {"replicas": 0, "template": {"spec": { + "serviceAccountName": "kars-controller", "automountServiceAccountToken": True, + "containers": [{"name": "controller", "image": "original", "env": [{"name": "X", "value": "Y"}]}], + "initContainers": [{"name": "init", "image": "original-init"}], + }}}} + before = copy.deepcopy(original) + result = safe_controller(original) + self.assertEqual(original, before) + pod = result["spec"]["template"]["spec"] + self.assertEqual(result["spec"]["replicas"], 1) + self.assertEqual(pod["serviceAccountName"], "kars-controller") + self.assertEqual(pod["schedulerName"], "kars-e2e-admission-never-schedule") + self.assertEqual(pod["containers"][0]["env"], [{"name": "X", "value": "Y"}]) + self.assertTrue(all(c["imagePullPolicy"] == "Never" and c["image"].startswith("registry.invalid/") + for c in pod["containers"] + pod["initContainers"])) + with self.assertRaises(AssertionError): + safe_controller({"kind": "Deployment", "metadata": {"name": "unrelated"}}) + + def test_chart_selection_never_executes_other_workloads_or_loads_secret_data(self): + rendered = "---\nkind: Secret\nmetadata:\n name: private\nstringData:\n token: do-not-publish\n" + rendered += "---\nkind: Job\nmetadata:\n name: execute\n---\nkind: ValidatingAdmissionPolicy\nmetadata:\n name: public\n" + result = builtin_documents(rendered) + self.assertNotIn("do-not-publish", result) + self.assertNotIn("kind: Job", result) + self.assertIn("kind: ValidatingAdmissionPolicy", result) + + def test_collection_tracks_real_uid_chain_without_logging_other_pods(self): + def request(_port, _method, path): + if path.endswith("/deployments"): + return 200, {"items": [{"metadata": {"name": "kars-controller", "uid": "dep"}}]} + if path.endswith("/replicasets"): + return 200, {"items": [{"metadata": {"name": "kars-controller-rs", "uid": "rs", + "ownerReferences": [{"uid": "dep"}]}, "status": {"replicas": 0}}]} + if path.endswith("/pods"): + return 200, {"items": [{"metadata": {"name": "created", "uid": "pod", + "ownerReferences": [{"uid": "rs"}]}}, + {"metadata": {"name": "unrelated", "uid": "other"}, "spec": {"private": "do-not-publish"}}]} + return 200, {"items": []} + result = collect(1, {}, request) + self.assertEqual([o["kind"] for o in result["workloads"]], ["Deployment", "ReplicaSet", "Pod"]) + self.assertNotIn("do-not-publish", json.dumps(result)) + self.assertNotIn("unrelated", json.dumps(result)) + + def test_failure_diagnostics_precede_teardown_without_pod_spec_or_log_dump(self): + source = (Path(__file__).resolve().parents[1] / "run.sh").read_text() + install = source.split("install_crds() {", 1)[1].split("\nteardown()", 1)[0] + self.assertIn("sre_authority.bootstrap_probe --diagnostics-only", install) + self.assertNotIn("kubectl describe pod", install) + self.assertNotIn("kubectl logs", install) + self.assertIn("return 1", install) + + +if __name__ == "__main__": + unittest.main() From 9a3921c3e1f9f8047ef6fb16f368de2de7fc7f32 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 19:49:40 +0200 Subject: [PATCH 12/62] test(sre): parse kubectl multi-object diagnostic output Handle adjacent JSON objects as well as Lists and expose only fixed diagnostic stages before parsing. No policy or runtime behavior changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/bootstrap_probe.py | 27 ++++++++++++++----- .../e2e/sre_authority/bootstrap_probe_test.py | 10 ++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index ffd8a62b7..534d2dfbd 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -36,7 +36,21 @@ def builtin_documents(rendered): return "\n---\n".join(documents) +def converted_objects(raw): + # kubectl create -o json emits adjacent JSON objects for a multi-document + # input, rather than the List returned by kubectl get. + decoder, objects = json.JSONDecoder(), [] + while raw.strip(): + obj, end = decoder.raw_decode(raw.lstrip()) + raw = raw.lstrip()[end:] + objects.extend(obj.get("items", []) if obj.get("kind") == "List" else [obj]) + if not objects or any(obj.get("kind") not in PATHS for obj in objects): + raise RuntimeError("Public bootstrap chart contained unexpected converted objects") + return objects + + def chart(root): + write_report(root, "bootstrap-stage.json", {"stage": "render-disabled-core"}) rendered = command("bootstrap-render", [ "helm", "template", "kars", str(root / "deploy/helm/kars"), "--namespace", "kars-system", # Offline rendering cannot satisfy the deliberate live-source Helm lookup. @@ -48,14 +62,13 @@ def chart(root): "--set-string", "foundry.endpoint=https://e2e-fake.invalid/", "--set-string", "foundry.projectEndpoint=https://e2e-fake.invalid/", ], root=root) - converted = json.loads(command("bootstrap-conversion", [ + write_report(root, "bootstrap-stage.json", {"stage": "convert-public-builtins"}) + converted = command("bootstrap-conversion", [ "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", - ], root=root, data=builtin_documents(rendered))) - objects = converted.get("items", []) if converted.get("kind") == "List" else [converted] - if not objects or any(obj.get("kind") not in PATHS for obj in objects): - raise RuntimeError("Public bootstrap chart contained unexpected converted objects") - return objects + ], root=root, data=builtin_documents(rendered)) + write_report(root, "bootstrap-stage.json", {"stage": "parse-public-objects"}) + return converted_objects(converted) def safe_controller(obj): @@ -148,12 +161,12 @@ def exercise(root, port, objects, policies): def main(root, diagnostics_only): with kind_proxy(root) as (port, version): + write_report(root, "bootstrap-versions.json", {"apiServer": version, "context": CONTEXT}) objects = chart(root) policies = {obj["metadata"]["name"]: obj for obj in objects if obj["kind"] == "ValidatingAdmissionPolicy"} if not policies: raise RuntimeError("Public admission policies were not rendered") - write_report(root, "bootstrap-versions.json", {"apiServer": version, "context": CONTEXT}) if diagnostics_only: write_report(root, "bootstrap-install-failure.json", collect(port, policies, request)) else: diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index a0df704f9..73cd2c954 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from .bootstrap_diagnostics import api_result, collect, failure_facts, object_status, policy_status -from .bootstrap_probe import builtin_documents, safe_controller +from .bootstrap_probe import builtin_documents, converted_objects, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ {"message": "Private SRE material requires authority"}]}}} @@ -73,6 +73,14 @@ def test_chart_selection_never_executes_other_workloads_or_loads_secret_data(sel self.assertNotIn("kind: Job", result) self.assertIn("kind: ValidatingAdmissionPolicy", result) + def test_kubectl_multiple_json_objects_and_list_are_both_parsed(self): + first = {"kind": "ServiceAccount", "metadata": {"name": "kars-controller"}} + second = {"kind": "ValidatingAdmissionPolicy", "metadata": {"name": "public"}} + self.assertEqual(converted_objects(json.dumps(first) + "\n" + json.dumps(second)), [first, second]) + self.assertEqual(converted_objects(json.dumps({"kind": "List", "items": [first, second]})), [first, second]) + with self.assertRaises(RuntimeError): + converted_objects(json.dumps({"kind": "Secret", "data": "do-not-publish"})) + def test_collection_tracks_real_uid_chain_without_logging_other_pods(self): def request(_port, _method, path): if path.endswith("/deployments"): From cc3c008635151277f18045b63ffbe81cce8e99ab Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 19:56:10 +0200 Subject: [PATCH 13/62] test(sre): retain fatal policy wait while collecting Pod evidence A missing observation on an unrelated resource policy must not suppress controller Pod creation diagnostics. Wait first, collect actual admission evidence, and still fail the observation gate. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/bootstrap_probe.py | 11 ++++++++-- .../e2e/sre_authority/bootstrap_probe_test.py | 22 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 534d2dfbd..f85cb791c 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -131,7 +131,11 @@ def exercise(root, port, objects, policies): time.sleep(1) else: write_report(root, "bootstrap-before-create.json", snapshot) - raise RuntimeError("Public admission policy observation timed out") + # Still collect actual Pod/ReplicaSet admission evidence after the + # bounded observation wait. An unrelated policy must not hide the + # creation failure; the unobserved-policy gate remains fatal below. + observed = all(entry.get("typeChecked") and entry.get("generation") == entry.get("observedGeneration") + for entry in snapshot["policies"]) write_report(root, "bootstrap-before-create.json", snapshot) templates = [obj for obj in objects if obj["kind"] == "Deployment" and obj["metadata"]["name"] == "kars-controller"] @@ -156,7 +160,10 @@ def exercise(root, port, objects, policies): if code != 201 or not result.get("accepted") or not any(obj.get("kind") == "Pod" for obj in snapshot["workloads"]): raise RuntimeError("Real controller Deployment/ReplicaSet did not create an admission-only Pod") write_report(root, "bootstrap-result.json", {"podCreation": "accepted", - "workloadExecution": "not-attempted", "readiness": "not-claimed"}) + "workloadExecution": "not-attempted", "readiness": "not-claimed", + "allPoliciesObservedBeforeCreate": observed}) + if not observed: + raise RuntimeError("Public admission policy observation timed out; Pod evidence was still collected") def main(root, diagnostics_only): diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 73cd2c954..36a78f8df 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -5,9 +5,10 @@ import json from pathlib import Path import unittest +from unittest.mock import patch from .bootstrap_diagnostics import api_result, collect, failure_facts, object_status, policy_status -from .bootstrap_probe import builtin_documents, converted_objects, safe_controller +from .bootstrap_probe import builtin_documents, converted_objects, exercise, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ {"message": "Private SRE material requires authority"}]}}} @@ -98,6 +99,25 @@ def request(_port, _method, path): self.assertNotIn("do-not-publish", json.dumps(result)) self.assertNotIn("unrelated", json.dumps(result)) + def test_observation_timeout_collects_pod_evidence_but_still_fails(self): + deployment = {"kind": "Deployment", "metadata": {"name": "kars-controller"}, + "spec": {"template": {"metadata": {"labels": {}}, "spec": {"containers": []}}}} + unobserved = {"policies": [{"generation": 1, "typeChecked": False}], "workloads": []} + created = dict(unobserved, workloads=[{"kind": "Pod"}]) + with patch("sre_authority.bootstrap_probe.request", return_value=(201, {})) as api, \ + patch("sre_authority.bootstrap_probe.upsert", return_value={"accepted": True}), \ + patch("sre_authority.bootstrap_probe.collect", side_effect=[unobserved, created]), \ + patch("sre_authority.bootstrap_probe.time.monotonic", side_effect=[0, 1, 91, 100, 101]), \ + patch("sre_authority.bootstrap_probe.time.sleep"), \ + patch("sre_authority.bootstrap_probe.write_report") as report: + with self.assertRaisesRegex(RuntimeError, "observation timed out"): + exercise(Path("."), 1, [deployment], POLICIES) + self.assertTrue(any(call.args[2].endswith("pods?dryRun=All") for call in api.call_args_list)) + result = next(call.args[2] for call in report.call_args_list if call.args[1] == "bootstrap-result.json") + self.assertEqual(result["podCreation"], "accepted") + self.assertFalse(result["allPoliciesObservedBeforeCreate"]) + self.assertEqual(result["readiness"], "not-claimed") + def test_failure_diagnostics_precede_teardown_without_pod_spec_or_log_dump(self): source = (Path(__file__).resolve().parents[1] / "run.sh").read_text() install = source.split("install_crds() {", 1)[1].split("\nteardown()", 1)[0] From 436fa33c80e7aaba430e9656e56dd21b258e31c9 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 20:06:04 +0200 Subject: [PATCH 14/62] test(sre): capture control-plane health without log contents The actual API admits the public Pod and Deployment but the Deployment remains unobserved. Record controller-manager restart/exit metadata and only fixed panic categories plus public Go frame names, stripping all log contents and arguments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../sre_authority/bootstrap_diagnostics.py | 58 +++++++++++++++++++ tests/e2e/sre_authority/bootstrap_probe.py | 6 +- .../e2e/sre_authority/bootstrap_probe_test.py | 22 ++++++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 7eb75dce7..e4bb2c6ac 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -4,6 +4,7 @@ """Allowlisted public admission evidence; never dump Pod specs, argv or bodies.""" import re +import subprocess REASONS = { "Forbidden", "Invalid", "InternalError", "BadRequest", "NotFound", "AlreadyExists", @@ -94,6 +95,59 @@ def policy_status(obj, expected): "typeChecked": "typeChecking" in status, "warnings": warnings} +def control_plane_status(obj): + containers = [] + for container in obj.get("status", {}).get("containerStatuses", [])[:4]: + state = container.get("state", {}) + previous = container.get("lastState", {}).get("terminated", {}) + containers.append({"name": identifier(container.get("name")), + "ready": container.get("ready") is True, "restartCount": container.get("restartCount"), + "state": next((name for name in ("running", "waiting", "terminated") if name in state), "unknown"), + "previousExitCode": previous.get("exitCode"), + "previousReason": previous.get("reason") if previous.get("reason") in ("Error", "Completed", "OOMKilled") else None}) + return {"name": identifier(obj.get("metadata", {}).get("name")), "containers": containers} + + +def public_stack_facts(text): + """Only fixed panic classifications and public Go frame names, never log text/args.""" + facts = {"panicCategories": [], "publicFrames": []} + for line in text.splitlines()[-240:]: + line = line.strip() + if line.startswith("panic:") or line.startswith("fatal error:"): + category = next((name for name, phrase in [ + ("nil-pointer", "nil pointer dereference"), + ("index-out-of-range", "index out of range"), + ("interface-conversion", "interface conversion"), + ("concurrent-map-write", "concurrent map writes"), + ] if phrase in line), "panic-redacted") + facts["panicCategories"].append(category) + frame = re.match(r"^((?:k8s\.io/(?:kubernetes|apiserver|apiextensions-apiserver)|github\.com/google/cel-go)/[A-Za-z0-9_./@*()+-]+)\(", line) + if frame and frame[1] not in facts["publicFrames"]: + facts["publicFrames"].append(frame[1][:512]) + facts["panicCategories"] = sorted(set(facts["panicCategories"])) + facts["publicFrames"] = facts["publicFrames"][:40] + return facts + + +def controller_stack(context, name): + if name != "kube-controller-manager-kars-e2e-control-plane" or context != "kind-kars-e2e": + return {"available": False} + result = {"available": False, "current": {}, "previous": {}} + for previous in (False, True): + try: + command = ["kubectl", "--context", context, "--request-timeout=10s", "logs", + "-n", "kube-system", name, "--tail=240"] + if previous: + command.append("--previous") + output = subprocess.run(command, capture_output=True, text=True, timeout=15, check=False) + if output.returncode == 0: + result["available"] = True + result["previous" if previous else "current"] = public_stack_facts(output.stdout) + except (OSError, subprocess.TimeoutExpired): + pass + return result + + def collect(port, policies, request): report = {"scope": "controller-Pod-creation-only", "policies": [], "workloads": [], "events": []} for name in sorted(policies): @@ -126,4 +180,8 @@ def collect(port, policies, request): "reason": event.get("reason"), "count": event.get("count")} entry.update(failure_facts(event.get("message"), policies)) report["events"].append(entry) + code, body = request(port, "GET", "/api/v1/namespaces/kube-system/pods?labelSelector=component%3Dkube-controller-manager") + if code == 200 and isinstance(body, dict): + report["controlPlane"] = [control_plane_status(obj) for obj in body.get("items", [])[:4] + if obj.get("metadata", {}).get("name") == "kube-controller-manager-kars-e2e-control-plane"] return report diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index f85cb791c..8431d5376 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -11,7 +11,7 @@ import re import time -from sre_authority.bootstrap_diagnostics import api_result, collect +from sre_authority.bootstrap_diagnostics import api_result, collect, controller_stack from sre_authority.registration_schema import CONTEXT, command, kind_proxy, request, write_report PATHS = { @@ -176,11 +176,15 @@ def main(root, diagnostics_only): raise RuntimeError("Public admission policies were not rendered") if diagnostics_only: write_report(root, "bootstrap-install-failure.json", collect(port, policies, request)) + write_report(root, "bootstrap-controller-stack.json", controller_stack( + CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) else: try: exercise(root, port, objects, policies) finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) + write_report(root, "bootstrap-controller-stack.json", controller_stack( + CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) if __name__ == "__main__": diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 36a78f8df..bd940438f 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import patch -from .bootstrap_diagnostics import api_result, collect, failure_facts, object_status, policy_status +from .bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, public_stack_facts from .bootstrap_probe import builtin_documents, converted_objects, exercise, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ @@ -46,6 +46,26 @@ def test_type_warning_scope_is_known_public_policy_field_only(self): with self.assertRaises(AssertionError): policy_status(obj, POLICIES) + def test_control_plane_metadata_and_go_frames_never_publish_logs_or_arguments(self): + pod = {"metadata": {"name": "kube-controller-manager-kars-e2e-control-plane"}, + "spec": {"containers": [{"args": ["do-not-publish"]}]}, + "status": {"containerStatuses": [{"name": "kube-controller-manager", "ready": False, + "restartCount": 4, "state": {"waiting": {"message": "do-not-publish"}}, + "lastState": {"terminated": {"exitCode": 2, "reason": "Error", "message": "do-not-publish"}}}]}} + status = control_plane_status(pod) + self.assertEqual(status["containers"][0]["restartCount"], 4) + self.assertNotIn("do-not-publish", json.dumps(status)) + facts = public_stack_facts( + "panic: runtime error: invalid memory address or nil pointer dereference\n" + "token=do-not-publish request body do-not-publish\n" + "k8s.io/apiserver/pkg/admission/plugin/policy/validating.(*TypeChecker).Check(do-not-publish)\n" + "panic: do-not-publish\n") + self.assertIn("nil-pointer", facts["panicCategories"]) + self.assertIn("panic-redacted", facts["panicCategories"]) + self.assertEqual(facts["publicFrames"], [ + "k8s.io/apiserver/pkg/admission/plugin/policy/validating.(*TypeChecker).Check"]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + def test_no_scheduling_pulls_or_execution_but_original_controller_shape_remains(self): original = {"kind": "Deployment", "metadata": {"name": "kars-controller"}, "spec": {"replicas": 0, "template": {"spec": { From b19d59701b29833dd25e0fb065c4546044796f3e Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 20:16:26 +0200 Subject: [PATCH 15/62] test(sre): prove preserved JSON schema candidate on actual API After the unchanged production probe fails, test only the API-evidenced additionalProperties:true adapter candidate in disposable Kind. Preserve all admission policies, require actual controller Pod creation and exact tenant/private denials, and verify arbitrary JSON params survive. Production schemas and readiness gates remain unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 5 + tests/e2e/sre_authority/bootstrap_cases.py | 97 +++++++++++++++++++ tests/e2e/sre_authority/bootstrap_probe.py | 45 +++++++-- .../e2e/sre_authority/bootstrap_probe_test.py | 32 +++++- 4 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/sre_authority/bootstrap_cases.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e82f1dd68..5b4fe2434 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,7 +423,11 @@ jobs: id: sre_schema run: python3 tests/e2e/sre_authority/registration_schema.py --exercise - name: Prove controller Pod admission with all chart policies and no image execution + id: sre_bootstrap run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe + - name: Collect nil-schema adapter candidate evidence without weakening production gates + if: failure() && steps.sre_bootstrap.outcome == 'failure' + run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --json-params-candidate - name: Collect namespace-accessor candidate evidence without relaxing the failing production gate if: failure() && steps.sre_schema.outcome == 'failure' run: python3 tests/e2e/sre_authority/registration_schema.py --namespace-accessor-candidate --exercise @@ -439,6 +443,7 @@ jobs: e2e-sre-schema-diag/namespace-accessor-candidate.json e2e-sre-schema-diag/namespace-accessor-candidate-instances.json e2e-sre-schema-diag/bootstrap-*.json + e2e-sre-schema-diag/candidate-bootstrap-*.json if-no-files-found: warn retention-days: 7 - name: Remove disposable schema cluster diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py new file mode 100644 index 000000000..ebd93a45a --- /dev/null +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Fixed public admission cases for the image-free schema candidate.""" + +import copy +import json +from urllib.error import HTTPError +from urllib.request import Request, build_opener, ProxyHandler + +from .bootstrap_diagnostics import api_result +from .bootstrap_probe import upsert +from .registration_schema import request + +USER = "system:serviceaccount:e2e-sre-bootstrap:tenant" + + +def as_tenant(port, path, obj): + req = Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(obj).encode(), method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json", + "Impersonate-User": USER}) + try: + response = build_opener(ProxyHandler({})).open(req, timeout=15) + except HTTPError as error: + response = error + with response: + return response.code, json.loads(response.read(1024 * 1024)) + + +def admission_cases(port, policies): + code, _ = request(port, "POST", "/api/v1/namespaces", { + "apiVersion": "v1", "kind": "Namespace", "metadata": {"name": "e2e-sre-bootstrap"}}) + if code not in (201, 409): + raise RuntimeError("Disposable admission principal namespace unavailable") + setup = [ + {"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "tenant", "namespace": "e2e-sre-bootstrap"}}, + {"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "sandbox", "namespace": "kars-sre"}}, + {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", + "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, + "rules": [{"apiGroups": [""], "resources": ["pods"], "verbs": ["create"]}, + {"apiGroups": ["apps"], "resources": ["deployments", "replicasets"], "verbs": ["create"]}, + {"apiGroups": ["kars.azure.com"], "resources": ["karssreactions"], "verbs": ["create"]}]}, + {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", + "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "e2e-bootstrap-probe"}, + "subjects": [{"kind": "ServiceAccount", "name": "tenant", "namespace": "e2e-sre-bootstrap"}]}, + ] + for obj in setup: + if not upsert(port, obj, policies).get("accepted"): + raise RuntimeError("Disposable namespaced admission principal setup failed") + pod = {"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "e2e-ordinary", "namespace": "kars-sre"}, + "spec": {"serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]}} + cases = [("ordinary-tenant-pod", "/api/v1/namespaces/kars-sre/pods", pod, 201, None)] + private = copy.deepcopy(pod) + private["spec"]["volumes"] = [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}] + cases.append(("private-volume", "/api/v1/namespaces/kars-sre/pods", private, 403, "kars-sre-private-mounts")) + env = copy.deepcopy(pod) + env["spec"]["containers"][0]["envFrom"] = [{"secretRef": {"name": "sre-api-router-identity"}}] + cases.append(("private-env", "/api/v1/namespaces/kars-sre/pods", env, 403, "kars-sre-private-mounts")) + for kind, plural in (("Deployment", "deployments"), ("ReplicaSet", "replicasets")): + for is_private in (False, True): + obj = {"apiVersion": "apps/v1", "kind": kind, + "metadata": {"name": "e2e-template", "namespace": "kars-sre"}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-probe"}}, + "template": {"metadata": {"labels": {"app": "e2e-probe"}}, + "spec": copy.deepcopy((private if is_private else pod)["spec"])}}} + cases.append((f"{kind}-{'private' if is_private else 'ordinary'}", + f"/apis/apps/v1/namespaces/kars-sre/{plural}", obj, + 403 if is_private else 201, "kars-sre-private-workloads" if is_private else None)) + params = {"namespace": "example", "name": "demo", "replicas": 1, + "nested": {"array": [True, None, 1, "text"], "object": {"key": "value"}}} + action = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSREAction", + "metadata": {"name": "e2e-proposal", "namespace": "kars-sre"}, + "spec": {"action": {"type": "ScaleDeployment", "params": params}, + "approval": {"state": "Pending"}, "ttlMinutes": 5}} + cases.append(("pending-json-params-preserved", "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions", + action, 201, None)) + approved = copy.deepcopy(action) + approved["spec"]["approval"]["state"] = "Approved" + cases.append(("preapproved-action-denied", "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions", + approved, 403, "kars-sre-pending-proposals")) + reports = [] + for name, path, obj, expected, policy in cases: + code, response = as_tenant(port, path + "?dryRun=All", obj) + result = api_result(code, response, policies) + result.update({"case": name, "expectedStatus": expected, "matched": code == expected + and (policy is None or policy in result.get("policies", []))}) + if name == "pending-json-params-preserved": + result["paramsPreserved"] = response.get("spec", {}).get("action", {}).get("params") == params + result["matched"] = result["matched"] and result["paramsPreserved"] + reports.append(result) + return reports diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 8431d5376..c05937938 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -12,7 +12,13 @@ import time from sre_authority.bootstrap_diagnostics import api_result, collect, controller_stack -from sre_authority.registration_schema import CONTEXT, command, kind_proxy, request, write_report +from sre_authority.registration_schema import CONTEXT, command, kind_proxy, request, write_report as schema_report + +REPORT_PREFIX = "" + + +def write_report(root, filename, report): + schema_report(root, REPORT_PREFIX + filename, report) PATHS = { "CustomResourceDefinition": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions", @@ -86,6 +92,21 @@ def safe_controller(obj): return fixture +def preserved_json_candidate(objects): + candidate = copy.deepcopy(objects) + matches = [obj for obj in candidate if obj["kind"] == "CustomResourceDefinition" + and obj["metadata"]["name"] == "karssreactions.kars.azure.com"] + if len(matches) != 1: + raise RuntimeError("Expected one public SRE action schema") + schema = matches[0]["spec"]["versions"][0]["schema"]["openAPIV3Schema"] + params = schema["properties"]["spec"]["properties"]["action"]["properties"]["params"] + if params.get("type") != "object" or params.get("additionalProperties") is not True: + raise RuntimeError("API-evidenced boolean additionalProperties candidate precondition failed") + del params["additionalProperties"] + params["x-kubernetes-preserve-unknown-fields"] = True + return candidate + + def upsert(port, obj, policies): metadata = obj.get("metadata", {}) path = PATHS[obj["kind"]].format(namespace=metadata.get("namespace", "kars-system")) @@ -104,7 +125,7 @@ def upsert(port, obj, policies): return result -def exercise(root, port, objects, policies): +def exercise(root, port, objects, policies, wait_seconds=90): results = [] for name in ("kars-system", "kars-sre"): code, body = request(port, "POST", "/api/v1/namespaces", { @@ -122,7 +143,7 @@ def exercise(root, port, objects, policies): if not result.get("accepted"): write_report(root, "bootstrap-install.json", {"operations": results}) raise RuntimeError("Disposable public bootstrap admission install failed") - deadline = time.monotonic() + 90 + deadline = time.monotonic() + wait_seconds while time.monotonic() < deadline: snapshot = collect(port, policies, request) if all(entry.get("typeChecked") and entry.get("generation") == entry.get("observedGeneration") @@ -166,10 +187,14 @@ def exercise(root, port, objects, policies): raise RuntimeError("Public admission policy observation timed out; Pod evidence was still collected") -def main(root, diagnostics_only): +def main(root, diagnostics_only, candidate=False): + global REPORT_PREFIX + REPORT_PREFIX = "candidate-" if candidate else "" with kind_proxy(root) as (port, version): write_report(root, "bootstrap-versions.json", {"apiServer": version, "context": CONTEXT}) objects = chart(root) + if candidate: + objects = preserved_json_candidate(objects) policies = {obj["metadata"]["name"]: obj for obj in objects if obj["kind"] == "ValidatingAdmissionPolicy"} if not policies: @@ -180,7 +205,13 @@ def main(root, diagnostics_only): CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) else: try: - exercise(root, port, objects, policies) + exercise(root, port, objects, policies, wait_seconds=180 if candidate else 90) + if candidate: + from sre_authority.bootstrap_cases import admission_cases + cases = admission_cases(port, policies) + write_report(root, "bootstrap-admission-cases.json", {"cases": cases}) + if not all(case["matched"] for case in cases): + raise RuntimeError("Schema candidate failed intended ordinary/private admission outcomes") finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( @@ -191,9 +222,11 @@ def main(root, diagnostics_only): os.umask(0o077) parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--diagnostics-only", action="store_true") + parser.add_argument("--json-params-candidate", action="store_true", + help="Diagnose only the API-evidenced nil-schema adapter candidate; never edit production") args = parser.parse_args() try: - main(Path(__file__).resolve().parents[3], args.diagnostics_only) + main(Path(__file__).resolve().parents[3], args.diagnostics_only, args.json_params_candidate) except Exception as error: print(f"SRE-BOOTSTRAP-FAIL category={type(error).__name__}", flush=True) raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index bd940438f..b55317420 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -8,7 +8,7 @@ from unittest.mock import patch from .bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, public_stack_facts -from .bootstrap_probe import builtin_documents, converted_objects, exercise, safe_controller +from .bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ {"message": "Private SRE material requires authority"}]}}} @@ -102,6 +102,36 @@ def test_kubectl_multiple_json_objects_and_list_are_both_parsed(self): with self.assertRaises(RuntimeError): converted_objects(json.dumps({"kind": "Secret", "data": "do-not-publish"})) + def test_candidate_changes_only_known_public_json_params_representation(self): + params = {"type": "object", "additionalProperties": True, "description": "public"} + action = {"kind": "CustomResourceDefinition", "metadata": {"name": "karssreactions.kars.azure.com"}, + "spec": {"versions": [{"schema": {"openAPIV3Schema": {"properties": {"spec": {"properties": { + "action": {"properties": {"params": params}}, "approval": {"type": "object"} + }}}}}}]}} + policy = {"kind": "ValidatingAdmissionPolicy", "metadata": {"name": "kars-sre-pending-proposals"}, + "spec": {"validations": [{"expression": "object.spec.approval.state == 'Pending'"}]}} + source = [action, policy] + before = copy.deepcopy(source) + changed = preserved_json_candidate(source) + self.assertEqual(source, before) + self.assertEqual(changed[1], policy) + actual = changed[0]["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]["action"]["properties"]["params"] + self.assertEqual(actual, {"type": "object", "x-kubernetes-preserve-unknown-fields": True, "description": "public"}) + params["additionalProperties"] = {"type": "string"} + with self.assertRaises(RuntimeError): + preserved_json_candidate(source) + + def test_arbitrary_failure_is_not_security_proof_in_candidate_cases(self): + from .bootstrap_cases import admission_cases + with patch("sre_authority.bootstrap_cases.request", return_value=(201, {})), \ + patch("sre_authority.bootstrap_cases.upsert", return_value={"accepted": True}), \ + patch("sre_authority.bootstrap_cases.as_tenant", return_value=( + 403, {"kind": "Status", "reason": "Forbidden", "message": "unrelated do-not-publish"})): + cases = admission_cases(1, POLICIES) + self.assertTrue(cases) + self.assertFalse(any(case["matched"] for case in cases)) + self.assertNotIn("do-not-publish", json.dumps(cases)) + def test_collection_tracks_real_uid_chain_without_logging_other_pods(self): def request(_port, _method, path): if path.endswith("/deployments"): From 4e012ec7f9cece155c19a8f272f976a7ae2ff469 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 20:55:27 +0200 Subject: [PATCH 16/62] fix(sre): preserve action JSON without the Kubernetes nil-schema crash Use a field-local preserved-unknown object schema in both Rust generation and Helm instead of boolean additionalProperties. Actual Kubernetes 1.31 A/B evidence restores controller-manager and Pod creation while preserving nine ordinary/private admission and JSON round-trip outcomes; no policy is relaxed. Run the admission cases against the successful shipped schema and fix test imports for real harness discovery. Rust, schema drift and full migration qualification remain required in hosted CI before readiness. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- controller/src/kars_sre_action.rs | 41 +++++++++++++++++++ .../kars/templates/crd-karssreaction.yaml | 2 +- .../2026-09-08-sre-authority-prerequisite.md | 31 +++++++++++++- tests/e2e/sre_authority/bootstrap_probe.py | 11 +++-- .../e2e/sre_authority/bootstrap_probe_test.py | 6 +-- 5 files changed, 79 insertions(+), 12 deletions(-) diff --git a/controller/src/kars_sre_action.rs b/controller/src/kars_sre_action.rs index 344649adc..3fdee9bf8 100644 --- a/controller/src/kars_sre_action.rs +++ b/controller/src/kars_sre_action.rs @@ -140,9 +140,19 @@ pub struct ActionSpec { /// - ScaleDeployment: {namespace, name, replicas} /// - RolloutRestart: {namespace, kind, name} /// - DeletePod: {namespace, name} + #[schemars(schema_with = "action_params_schema")] pub params: std::collections::BTreeMap, } +fn action_params_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + // Boolean additionalProperties exposes a nil-schema adapter in Kubernetes + // 1.31's VAP type checker. Preserve arbitrary JSON without that adapter. + schemars::json_schema!({ + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }) +} + /// Operator decision payload. #[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq)] #[serde(rename_all = "camelCase")] @@ -192,3 +202,34 @@ pub struct KarsSREActionStatus { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub conditions: Vec, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sre_action_params_schema_preserves_json_without_boolean_additional_properties() { + let crd = serde_json::to_value(crate::crd_validations::kars_sre_action_crd()).unwrap(); + let spec = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let params = &spec["properties"]["action"]["properties"]["params"]; + assert_eq!(params["type"], "object"); + assert_eq!(params["x-kubernetes-preserve-unknown-fields"], true); + assert!(params.get("additionalProperties").is_none()); + assert!(spec.get("x-kubernetes-preserve-unknown-fields").is_none()); + assert_eq!( + spec["properties"]["approval"]["properties"]["state"]["type"], + "string" + ); + } + + #[test] + fn sre_action_params_schema_does_not_change_json_wire_values() { + let wire = serde_json::json!({ + "type": "ScaleDeployment", + "params": {"namespace": "example", "name": "demo", "replicas": 1, + "nested": {"array": [true, null, 1, "text"], "object": {"key": "value"}}} + }); + let action: ActionSpec = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(action).unwrap(), wire); + } +} diff --git a/deploy/helm/kars/templates/crd-karssreaction.yaml b/deploy/helm/kars/templates/crd-karssreaction.yaml index aba25b887..13b8abe13 100644 --- a/deploy/helm/kars/templates/crd-karssreaction.yaml +++ b/deploy/helm/kars/templates/crd-karssreaction.yaml @@ -56,7 +56,6 @@ spec: free-form params (validated per-type at reconcile time). properties: params: - additionalProperties: true description: |- Per-type params. Stored as a string-keyed map so the CRD schema emits a concrete `type: object` (apiserver rejects fields with @@ -70,6 +69,7 @@ spec: - RolloutRestart: {namespace, kind, name} - DeletePod: {namespace, name} type: object + x-kubernetes-preserve-unknown-fields: true type: description: |- Action type from the closed set (`DeleteResourceQuota`, diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 009b35ac1..2e907d802 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -1,7 +1,7 @@ # Security audit — registered SRE credential authority -Status: implemented and locally qualified candidate; pending real Kubernetes -admission/migration proof and independent review. **Not a sign-off.** +Status: candidate under qualification; the latest schema repair still requires +Rust and full controller/migration execution. **Not a sign-off.** ## Scope and trust root @@ -10,6 +10,33 @@ Only explicitly delegated registrars can author it; the controller can read, use, and reconcile status. Namespace occupancy, SRE labels, account names, and Helm-looking metadata are not privilege delegation. +## Kubernetes 1.31 controller-manager compatibility + +Real Kubernetes v1.31.0 evidence showed that the controller Pod was not rejected: +Pod dry-run and Deployment creation returned 201, but kube-controller-manager +repeatedly exited with a nil-pointer panic while the VAP status controller +converted the SRE action CRD's OpenAPI schema. The trigger was the boolean +`additionalProperties: true` representation of `spec.action.params`. + +The repair keeps `type: object` and arbitrary nested JSON values, using +`x-kubernetes-preserve-unknown-fields: true` for that field only. Rust schema +generation and the Helm CRD use the same representation. No admission policy, +approval requirement, namespace boundary or other field constraint is removed. + +The hosted A/B proof at +https://github.com/Azure/kars/actions/runs/34262068112/job/102182569705 +keeps the original failing step fatal. The separate schema-only candidate +recovered controller-manager, observed all 22 policies, created a real +Deployment/ReplicaSet/Pod, and passed nine ordinary/private admission and +Pending-action JSON round-trip cases. Workload scheduling, image execution and +application readiness were deliberately not claimed by that admission probe. + +The successful shipped-schema probe now runs those admission cases too. +Local Python discovery covers all 35 diagnostic/harness cases; its imports match +the actual full-harness discovery command. The new Rust schema/wire regressions, +existing Helm/Rust drift test and full fatal SRE migration remain required before +readiness. A CI run, not the unbuilt local repair, supplies that next evidence. + ## Boundaries implemented - Exact source, controller/release, and runtime namespace UIDs are checked live. diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index c05937938..bc8da60da 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -206,12 +206,11 @@ def main(root, diagnostics_only, candidate=False): else: try: exercise(root, port, objects, policies, wait_seconds=180 if candidate else 90) - if candidate: - from sre_authority.bootstrap_cases import admission_cases - cases = admission_cases(port, policies) - write_report(root, "bootstrap-admission-cases.json", {"cases": cases}) - if not all(case["matched"] for case in cases): - raise RuntimeError("Schema candidate failed intended ordinary/private admission outcomes") + from sre_authority.bootstrap_cases import admission_cases + cases = admission_cases(port, policies) + write_report(root, "bootstrap-admission-cases.json", {"cases": cases}) + if not all(case["matched"] for case in cases): + raise RuntimeError("Schema failed intended ordinary/private admission outcomes") finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index b55317420..77c4bb7e9 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,8 +7,8 @@ import unittest from unittest.mock import patch -from .bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, public_stack_facts -from .bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, public_stack_facts +from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ {"message": "Private SRE material requires authority"}]}}} @@ -122,7 +122,7 @@ def test_candidate_changes_only_known_public_json_params_representation(self): preserved_json_candidate(source) def test_arbitrary_failure_is_not_security_proof_in_candidate_cases(self): - from .bootstrap_cases import admission_cases + from sre_authority.bootstrap_cases import admission_cases with patch("sre_authority.bootstrap_cases.request", return_value=(201, {})), \ patch("sre_authority.bootstrap_cases.upsert", return_value={"accepted": True}), \ patch("sre_authority.bootstrap_cases.as_tenant", return_value=( From d9910d42beed2aa7bebad39119f93ff54c97d37f Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 21:53:52 +0200 Subject: [PATCH 17/62] test(sre): stabilize real token fixtures without minting credentials Bind synthetic token Secrets to real fixture ServiceAccount UIDs with complete public CA and namespace data, preventing healthy TokenController garbage collection or JWT generation. Preserve precise admission and immutable-type denials, prove unowned same-name identity quarantine and replacement, and retain primary failures during fenced cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/credential_paths.py | 171 ++++++++++--- .../sre_authority/credential_paths_test.py | 240 ++++++++++++++++++ tests/e2e/sre_authority/migration.py | 6 +- 3 files changed, 376 insertions(+), 41 deletions(-) create mode 100644 tests/e2e/sre_authority/credential_paths_test.py diff --git a/tests/e2e/sre_authority/credential_paths.py b/tests/e2e/sre_authority/credential_paths.py index 4f9be1e69..bfe51596a 100644 --- a/tests/e2e/sre_authority/credential_paths.py +++ b/tests/e2e/sre_authority/credential_paths.py @@ -3,19 +3,53 @@ import base64 from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager import copy import json import threading import time -from .common import AGENT, PRIVATE, REGISTRATION, RUNTIME, SYSTEM, assert_denial, require +from .common import AGENT, OWNER, PRIVATE, REGISTRATION, RUNTIME, SYSTEM, assert_denial, require WATCH_BINDING = "e2e-sre-legacy-watch-only" TOKEN_ALIAS = "e2e-prestaged-router-token" TOKEN_TYPE = "kubernetes.io/service-account-token" SA_NAME = "kubernetes.io/service-account.name" +SA_UID = "kubernetes.io/service-account.uid" +SYNTHETIC_TOKEN = "kind-synthetic-not-a-jwt-never-authenticates" WATCH_MARKER = "kind-dummy-secret-watch-proof" + +def synthetic_token_secret(h, name, account): + metadata = account["metadata"] + require(metadata.get("uid") and metadata.get("namespace") == RUNTIME + and not metadata.get("deletionTimestamp"), "Synthetic token fixture requires a live ServiceAccount UID") + ca = h.poll("public namespace CA for synthetic token fixture", + lambda: h.get("configmap", "kube-root-ca.crt", RUNTIME), seconds=30) + ca = ca.get("data", {}).get("ca.crt", "") + require(ca.startswith("-----BEGIN CERTIFICATE-----"), "Synthetic token fixture lacks the public cluster CA") + # A matching live SA UID and complete data prevent native TokenController + # garbage collection and token generation. Never authenticate with this data. + return {"apiVersion": "v1", "kind": "Secret", "type": TOKEN_TYPE, + "metadata": {"name": name, "namespace": RUNTIME, + "annotations": {SA_NAME: metadata["name"], SA_UID: metadata["uid"]}}, + "data": {key: base64.b64encode(value.encode()).decode() for key, value in { + "token": SYNTHETIC_TOKEN, "ca.crt": ca, "namespace": RUNTIME}.items()}} + + +def assert_secret_unchanged(current, before): + require(current is not None, "Token Secret fixture disappeared; absence is not denial proof") + require(all(current["metadata"].get(key) == before["metadata"].get(key) + for key in ("name", "namespace", "uid", "resourceVersion", "annotations", "labels", + "ownerReferences", "finalizers", "deletionTimestamp")) + and current.get("type") == before.get("type") + and current.get("data", {}) == before.get("data", {}), + "Token Secret fixture was replaced, mutated or populated") + if current.get("type") == TOKEN_TYPE: + require(current.get("data", {}).get("token") == base64.b64encode(SYNTHETIC_TOKEN.encode()).decode(), + "Token Secret fixture no longer contains the exact synthetic non-JWT") + + def assert_token_type_immutable(response, label): """Secret type validation precedes VAP; this is not admission-policy proof.""" require(response.status_code == 422, @@ -31,10 +65,14 @@ def assert_token_type_immutable(response, label): def seed_privacy_gaps(h): require(h.get("serviceaccount", "sre-api-router", RUNTIME) is None, - "Legacy token alias must precede private ServiceAccount creation") - alias = h.create({"apiVersion": "v1", "kind": "Secret", "type": TOKEN_TYPE, - "metadata": {"name": TOKEN_ALIAS, "namespace": RUNTIME, "annotations": {SA_NAME: "sre-api-router"}}}) - require(not alias.get("data"), "Prestaged token alias unexpectedly contains credentials") + "Legacy fixtures require an unoccupied reserved ServiceAccount name") + account = h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": "sre-api-router", "namespace": RUNTIME}, + "automountServiceAccountToken": False}) + h.state["prestaged_account"] = account + h.state["prestaged_account_cleaned"] = False + alias = h.create(synthetic_token_secret(h, TOKEN_ALIAS, account)) + assert_secret_unchanged(h.get("secret", TOKEN_ALIAS, RUNTIME), alias) h.state["prestaged_alias"] = alias h.create({"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", "metadata": {"name": WATCH_BINDING}, @@ -48,33 +86,65 @@ def seed_privacy_gaps(h): h.create({"apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": "e2e-watch-only", "namespace": RUNTIME}}) h.token_identity("watch-only", RUNTIME, "e2e-watch-only") - h.passed("Watch-only group grant and arbitrary token alias predate admission and private ServiceAccount creation") + h.passed("Pre-guard same-name ServiceAccount has no controller-issued authority; its stable token alias contains only synthetic non-JWT data") + h.passed("Unsafe legacy group grants intentionally cover all runtime ServiceAccounts; no token is requested for the same-name fixture") def delete_owned(h, path, obj): - h.api("DELETE", path, body={"apiVersion": "v1", "kind": "DeleteOptions", + response = h.api("DELETE", path, body={"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": obj["metadata"]["uid"], - "resourceVersion": obj["metadata"]["resourceVersion"]}}, status=(200, 202)) + "resourceVersion": obj["metadata"]["resourceVersion"]}}, status=(200, 202, 404)) + if response.status_code == 404: + require(response.json().get("kind") == "Status" and response.json().get("reason") == "NotFound", + "Fixture cleanup did not return a Kubernetes NotFound status") + + +@contextmanager +def owned_fixtures(h): + fixtures = [] + primary_failure = False + try: + yield fixtures + except BaseException: + primary_failure = True + raise + finally: + failed = False + for path, obj in reversed(fixtures): + try: + delete_owned(h, path, obj) + except Exception: + failed = True + if failed: + if primary_failure: + print("SRE-DIAG Fenced fixture cleanup failed; retaining the primary acceptance failure", flush=True) + else: + raise AssertionError("Fixture cleanup failed; UID/resourceVersion fences were not retried or weakened") def token_secret_denials(h, before_enrollment=False): policy = "kars-sre-no-legacy-tokens" path = f"/api/v1/namespaces/{RUNTIME}/secrets" prefix = f"e2e-token-{h.phase}-{'pre' if before_enrollment else 'live'}" - unsafe = {"apiVersion": "v1", "kind": "Secret", "type": TOKEN_TYPE, - "metadata": {"name": f"{prefix}-create", "namespace": RUNTIME, - "annotations": {SA_NAME: "sre-api-router"}}} + account = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(account is not None, "Token denial probes require their actual same-name ServiceAccount fixture") + unsafe = synthetic_token_secret(h, f"{prefix}-create", account) # Real CREATE/PATCH, not a fixed private name or TokenRequest. A broken - # policy fails the disposable cluster immediately, without reading tokens. - for user in ("tenant", "registrar"): - assert_denial(h.api("POST", path, body=unsafe, user=user), - f"{user} arbitrary token Secret CREATE", policy) - fixtures = [] - try: + # policy fails immediately; even an accepted fixture cannot mint a JWT. + with owned_fixtures(h) as fixtures: + for user in ("tenant", "registrar"): + response = h.api("POST", path, body=unsafe, user=user) + if response.status_code == 201: + fixtures.append((path + "/" + unsafe["metadata"]["name"], response.json())) + assert_denial(response, f"{user} arbitrary token Secret CREATE", policy) + benign = h.create({"apiVersion": "v1", "kind": "ServiceAccount", + "metadata": {"name": f"{prefix}-account", "namespace": RUNTIME}, + "automountServiceAccountToken": False}) + fixtures.append((f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/{prefix}-account", benign)) variants = [ ("type-and-annotation", "Opaque", {}, {"type": TOKEN_TYPE, "metadata": {"annotations": {SA_NAME: "sre-api-router"}}}), - ("annotation", TOKEN_TYPE, {SA_NAME: "e2e-never-created-account"}, + ("annotation", TOKEN_TYPE, {}, {"metadata": {"annotations": {SA_NAME: "sre-api-router"}}}), ] if before_enrollment: @@ -83,10 +153,14 @@ def token_secret_denials(h, before_enrollment=False): # be introduced into an already-Ready runtime even for this test. variants.append(("type", "Opaque", {SA_NAME: "sre-api-router"}, {"type": TOKEN_TYPE})) for suffix, kind, annotations, patch in variants: - obj = h.api("POST", path, body={"apiVersion": "v1", "kind": "Secret", "type": kind, + body = {"apiVersion": "v1", "kind": "Secret", "type": kind, "metadata": {"name": f"{prefix}-{suffix}", "namespace": RUNTIME, - "annotations": annotations}}, user="tenant", status=201).json() - fixtures.append(obj) + "annotations": annotations}} + if kind == TOKEN_TYPE: + body = synthetic_token_secret(h, f"{prefix}-{suffix}", benign) + obj = h.api("POST", path, body=body, user="tenant", status=201).json() + fixtures.append((path + "/" + obj["metadata"]["name"], obj)) + assert_secret_unchanged(h.get("secret", obj["metadata"]["name"], RUNTIME), obj) response = h.api("PATCH", path + "/" + obj["metadata"]["name"], body={ **patch, "metadata": {**patch.get("metadata", {}), "uid": obj["metadata"]["uid"], "resourceVersion": obj["metadata"]["resourceVersion"]}}, @@ -95,22 +169,19 @@ def token_secret_denials(h, before_enrollment=False): assert_token_type_immutable(response, f"token Secret {suffix} PATCH") else: assert_denial(response, f"token Secret {suffix} PATCH", policy) - current = h.get("secret", obj["metadata"]["name"], RUNTIME) - require(current["metadata"]["resourceVersion"] == obj["metadata"]["resourceVersion"] - and not current.get("data"), "Denied token Secret update mutated or populated its fixture") + assert_secret_unchanged(h.get("secret", obj["metadata"]["name"], RUNTIME), obj) if before_enrollment: for patch in ({"type": "Opaque"}, {"metadata": {"annotations": {SA_NAME: "e2e-other"}}}): before = h.get("secret", TOKEN_ALIAS, RUNTIME) - response = h.api("PATCH", path + "/" + TOKEN_ALIAS, body=patch, user="tenant") + assert_secret_unchanged(before, h.state["prestaged_alias"]) + response = h.api("PATCH", path + "/" + TOKEN_ALIAS, body={ + **patch, "metadata": {**patch.get("metadata", {}), "uid": before["metadata"]["uid"], + "resourceVersion": before["metadata"]["resourceVersion"]}}, user="tenant") if "type" in patch: assert_token_type_immutable(response, "prestaged oldObject type escape") else: assert_denial(response, "prestaged oldObject annotation escape", policy) - require(h.get("secret", TOKEN_ALIAS, RUNTIME) == before, - "Rejected oldObject token escape changed its prestaged fixture") - finally: - for obj in fixtures: - delete_owned(h, path + "/" + obj["metadata"]["name"], obj) + assert_secret_unchanged(h.get("secret", TOKEN_ALIAS, RUNTIME), before) h.passed("Tenant and registrar token Secret CREATE and schema-valid annotation updates receive exact admission denials") h.passed("Token-type mutations receive specific Kubernetes immutable-field errors without changing fixtures") @@ -155,14 +226,29 @@ def assert_unissued(h, spec, before): "Blocked migration changed a legacy grant") require(h.get("deployment", "sre", RUNTIME)["spec"]["replicas"] == 1, "Blocked migration stopped the legacy consumer") - require(h.get("serviceaccount", "sre-api-router", RUNTIME) is None, - "Private ServiceAccount appeared while legacy privacy was unsafe") + expected = None if h.state["prestaged_account_cleaned"] else h.state["prestaged_account"] + require(h.get("serviceaccount", "sre-api-router", RUNTIME) == expected, + "Controller adopted, mutated or replaced the unowned same-name ServiceAccount before safe enrollment") require(all(h.get("secret", name, RUNTIME) is None for name in (PRIVATE, AGENT)), "Private credential appeared while legacy privacy was unsafe") for name in ("kars-sre-private-reader", "kars-sre-private-author", "kars-sre-private-renew"): require(h.get("clusterrolebinding", name) is None, "Private grant appeared while legacy privacy was unsafe") require(h.get("rolebinding", "sre-api-self-renew", RUNTIME) is None, "Private token-renewal grant appeared while legacy privacy was unsafe") + reg = h.get("karssreregistrations.kars.azure.com", "canonical") + require(not reg.get("status", {}).get("routerServiceAccountUid"), + "Controller registered the unowned same-name ServiceAccount as private authority") + + +def assert_fresh_private_identity(h, reg): + account = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(h.state["prestaged_account_cleaned"] and account + and account["metadata"]["uid"] != h.state["prestaged_account"]["metadata"]["uid"] + and account["metadata"]["uid"] == reg["status"].get("routerServiceAccountUid") + and account["metadata"].get("annotations", {}).get(OWNER) == reg["metadata"]["uid"] + and account.get("automountServiceAccountToken") is False, + "Trusted private identity was not freshly controller-created after unowned fixture cleanup") + require(h.get("secret", TOKEN_ALIAS, RUNTIME) is None, "Prestaged token alias survived private issuance") def block_prestaged_paths(h, spec, before): @@ -172,10 +258,18 @@ def block_prestaged_paths(h, spec, before): wait_blocked(h, "Unsafe legacy SRE token Secret alias") assert_unissued(h, spec, before) alias = h.get("secret", TOKEN_ALIAS, RUNTIME) - require(alias == h.state["prestaged_alias"] and not alias.get("data"), - "Prestaged token Secret was deleted, adopted, populated or modified") + assert_secret_unchanged(alias, h.state["prestaged_alias"]) delete_owned(h, f"/api/v1/namespaces/{RUNTIME}/secrets/{TOKEN_ALIAS}", alias) - h.passed("Real controller quarantines the untouched prestaged token alias before private identity/grants/issuance; operator CAS cleanup only") + h.poll("prestaged synthetic alias cleanup", lambda: h.get("secret", TOKEN_ALIAS, RUNTIME) is None, seconds=30) + # The broad watch grant still blocks enrollment while the operator removes + # the unowned same-name SA. Never adopt it by name or attach private grants. + account = h.get("serviceaccount", "sre-api-router", RUNTIME) + require(account == h.state["prestaged_account"], "Unowned ServiceAccount changed before operator cleanup") + delete_owned(h, f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router", account) + h.poll("unowned same-name ServiceAccount cleanup", + lambda: h.get("serviceaccount", "sre-api-router", RUNTIME) is None, seconds=30) + h.state["prestaged_account_cleaned"] = True + h.passed("Real controller blocks the unchanged synthetic alias before adopting the same-name SA or issuing private authority; operator UID/RV-fenced cleanup removes both") wait_blocked(h, "broad group grant") assert_unissued(h, spec, before) @@ -232,15 +326,12 @@ def observe(): return response, True return response, False - obj = None - try: + with owned_fixtures(h) as fixtures: with ThreadPoolExecutor(max_workers=1) as pool: result = pool.submit(observe) require(started.wait(timeout=3), "Secret watch probe did not start") obj = h.create({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name, "namespace": namespace}, "stringData": {"dummy": WATCH_MARKER}}) + fixtures.append((f"/api/v1/namespaces/{namespace}/secrets/{name}", obj)) response, observed = result.result(timeout=12) assert_watch_result(response, observed, allowed) - finally: - if obj: - delete_owned(h, f"/api/v1/namespaces/{namespace}/secrets/{name}", obj) diff --git a/tests/e2e/sre_authority/credential_paths_test.py b/tests/e2e/sre_authority/credential_paths_test.py new file mode 100644 index 000000000..e70949373 --- /dev/null +++ b/tests/e2e/sre_authority/credential_paths_test.py @@ -0,0 +1,240 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Fixture contract checks, not substitutes for hosted Kubernetes acceptance.""" + +import base64 +import copy +import io +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +from sre_authority.common import OWNER, RUNTIME, require +from sre_authority.credential_paths import ( + SA_NAME, SA_UID, SYNTHETIC_TOKEN, TOKEN_ALIAS, TOKEN_TYPE, + assert_fresh_private_identity, assert_secret_unchanged, assert_unissued, + delete_owned, owned_fixtures, seed_privacy_gaps, synthetic_token_secret, token_secret_denials, +) +from sre_authority.harness_test import Response + +POLICY = "kars-sre-no-legacy-tokens" +CA = "-----BEGIN CERTIFICATE-----\npublic-test-ca\n-----END CERTIFICATE-----\n" +SECRET_PATH = f"/api/v1/namespaces/{RUNTIME}/secrets" + + +class FixtureHarness: + def __init__(self): + self.state = {} + self.phase = "legacy" + self.objects = {} + self.requests = [] + self.identities = [] + self.sequence = 0 + + def create(self, obj): + obj = copy.deepcopy(obj) + self.sequence += 1 + obj["metadata"].update(uid=f"uid-{self.sequence}", resourceVersion=str(self.sequence)) + self.objects[obj["kind"].lower(), obj["metadata"]["name"]] = obj + return copy.deepcopy(obj) + + def get(self, kind, name, _namespace=None): + if kind == "configmap" and name == "kube-root-ca.crt": + return {"data": {"ca.crt": CA}} + if kind == "karssreregistrations.kars.azure.com": + kind = "karssreregistration" + return copy.deepcopy(self.objects.get((kind, name))) + + def poll(self, _label, predicate, **_kwargs): + result = predicate() + require(result, "Unit fixture did not satisfy the real harness predicate") + return result + + def passed(self, _message): + pass + + def token_identity(self, *args): + self.identities.append(args) + + def api(self, method, path, *, body=None, user="admin", status=None): + self.requests.append((method, path, copy.deepcopy(body), user)) + resource = path.split("/")[-2] + kind = {"secrets": "secret", "serviceaccounts": "serviceaccount"}.get(resource) + key = (kind, path.split("/")[-1]) + current = self.objects.get(key) + if method == "POST": + private = (body.get("type") == TOKEN_TYPE + and body["metadata"].get("annotations", {}).get(SA_NAME) == "sre-api-router") + response = Response(403, {"kind": "Status", "reason": "Forbidden", "message": POLICY}) if private \ + else Response(201, self.create(body)) + elif method == "PATCH": + if current is None: + response = Response(404, {"kind": "Status", "reason": "NotFound"}) + elif "type" in body: + response = Response(422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [ + {"field": "type", "reason": "FieldValueInvalid", "message": "field is immutable"}]}}) + else: + response = Response(403, {"kind": "Status", "reason": "Forbidden", "message": POLICY}) + elif method == "DELETE": + if current is None: + response = Response(404, {"kind": "Status", "reason": "NotFound"}) + elif body["preconditions"] != {k: current["metadata"][k] for k in ("uid", "resourceVersion")}: + response = Response(409, {"kind": "Status", "reason": "Conflict"}) + else: + del self.objects[key] + response = Response(200, {}) + else: + raise AssertionError("Unexpected unit fixture request") + if status is not None: + require(response.status_code in (status if isinstance(status, tuple) else (status,)), + f"Unit API HTTP {response.status_code}") + return response + + +class CredentialFixtureTests(unittest.TestCase): + def seeded(self): + h = FixtureHarness() + seed_privacy_gaps(h) + return h + + def test_seed_uses_live_unowned_account_and_complete_noncredential_data(self): + h = self.seeded() + account = h.state["prestaged_account"] + secret = h.state["prestaged_alias"] + self.assertFalse(account["automountServiceAccountToken"]) + for field in ("ownerReferences", "annotations", "labels", "finalizers"): + self.assertNotIn(field, account["metadata"]) + self.assertNotIn("secrets", account) + self.assertEqual(secret["metadata"]["annotations"], { + SA_NAME: "sre-api-router", SA_UID: account["metadata"]["uid"]}) + self.assertEqual({key: base64.b64decode(value).decode() for key, value in secret["data"].items()}, + {"token": SYNTHETIC_TOKEN, "ca.crt": CA, "namespace": RUNTIME}) + self.assertNotIn(".", SYNTHETIC_TOKEN) + self.assertNotIn("finalizers", secret["metadata"]) + self.assertEqual(h.identities, [("watch-only", RUNTIME, "e2e-watch-only")]) + + def test_synthetic_secret_requires_current_account_uid_namespace_and_ca(self): + h = self.seeded() + account = h.state["prestaged_account"] + for changed in ({"uid": ""}, {"namespace": "other"}, {"deletionTimestamp": "now"}): + bad = copy.deepcopy(account) + bad["metadata"].update(changed) + with self.subTest(changed=changed), self.assertRaises(AssertionError): + synthetic_token_secret(h, "probe", bad) + with patch.object(h, "get", return_value={"data": {}}), self.assertRaises(AssertionError): + synthetic_token_secret(h, "probe", account) + + def test_missing_replaced_modified_or_populated_fixture_never_proves_denial(self): + secret = self.seeded().state["prestaged_alias"] + assert_secret_unchanged(copy.deepcopy(secret), secret) + variants = [None] + for key in ("uid", "resourceVersion", "annotations", "finalizers", "ownerReferences"): + bad = copy.deepcopy(secret) + bad["metadata"][key] = "changed" + variants.append(bad) + for changed in ({"data": {}}, {"data": {"token": "unexpected"}}, {"type": "Opaque"}): + variants.append({**secret, **changed}) + for bad in variants: + with self.subTest(case=variants.index(bad)), self.assertRaises(AssertionError): + assert_secret_unchanged(bad, secret) + + def test_real_probe_contract_preserves_old_object_and_uses_live_benign_sa(self): + for before in (True, False): + with self.subTest(before=before): + h = self.seeded() + token_secret_denials(h, before_enrollment=before) + posts = [body for method, path, body, _ in h.requests + if method == "POST" and path == SECRET_PATH] + tokens = [obj for obj in posts if obj["type"] == TOKEN_TYPE] + self.assertEqual(len(tokens), 3) # Two CREATE denials and the annotation fixture. + for obj in tokens: + self.assertTrue(obj["metadata"]["annotations"][SA_UID]) + self.assertEqual(base64.b64decode(obj["data"]["token"]).decode(), SYNTHETIC_TOKEN) + annotation = next(obj for obj in tokens if obj["metadata"]["name"].endswith("-annotation")) + self.assertNotEqual(annotation["metadata"]["annotations"][SA_NAME], "sre-api-router") + escapes = [body for method, path, body, _ in h.requests + if method == "PATCH" and path.endswith("/" + TOKEN_ALIAS)] + self.assertEqual(len(escapes), 2 if before else 0) + for body in escapes: + self.assertEqual(body["metadata"]["uid"], h.state["prestaged_alias"]["metadata"]["uid"]) + self.assertEqual(body["metadata"]["resourceVersion"], + h.state["prestaged_alias"]["metadata"]["resourceVersion"]) + self.assertEqual(h.get("secret", TOKEN_ALIAS), h.state["prestaged_alias"]) + self.assertFalse(any(name.startswith("e2e-token-") for _, name in h.objects)) + + def test_cleanup_accepts_only_real_absence_and_never_drops_cas_fences(self): + h = self.seeded() + obj = h.state["prestaged_alias"] + path = SECRET_PATH + "/" + TOKEN_ALIAS + delete_owned(h, path, obj) + delete_owned(h, path, obj) # Already absent is cleanup, never admission proof. + self.assertEqual(h.requests[-1][2]["preconditions"], { + key: obj["metadata"][key] for key in ("uid", "resourceVersion")}) + replacement = h.create({**obj, "metadata": {"name": TOKEN_ALIAS, "namespace": RUNTIME}}) + with self.assertRaisesRegex(AssertionError, "409"): + delete_owned(h, path, obj) + self.assertEqual(h.get("secret", TOKEN_ALIAS), replacement) + with patch.object(h, "api", return_value=Response(404, {})), self.assertRaises(AssertionError): + delete_owned(h, path, obj) + + def test_cleanup_conflict_cannot_mask_primary_error_and_all_fixtures_are_attempted(self): + h = self.seeded() + output = io.StringIO() + with patch("sre_authority.credential_paths.delete_owned", side_effect=AssertionError("private-body")) as cleanup: + with redirect_stdout(output), self.assertRaisesRegex(AssertionError, "^primary denial failure$"): + with owned_fixtures(h) as fixtures: + fixtures.extend([("first", {}), ("second", {})]) + raise AssertionError("primary denial failure") + self.assertEqual([call.args[1] for call in cleanup.call_args_list], ["second", "first"]) + self.assertNotIn("private-body", output.getvalue()) + with self.assertRaisesRegex(AssertionError, "Fixture cleanup failed"): + with owned_fixtures(h) as fixtures: + fixtures.append(("first", {})) + + def test_disappeared_fixture_patch_404_is_fatal_even_when_cleanup_is_404(self): + h = self.seeded() + original = h.api + def missing(method, path, **kwargs): + if method == "PATCH": + h.objects.pop(("secret", path.split("/")[-1]), None) + return Response(404, {"kind": "Status", "reason": "NotFound"}) + return original(method, path, **kwargs) + with patch.object(h, "api", side_effect=missing), self.assertRaisesRegex(AssertionError, "got HTTP 404"): + token_secret_denials(h, before_enrollment=True) + + def test_blocked_controller_cannot_adopt_or_record_same_name_fixture(self): + h = self.seeded() + h.create({"kind": "Deployment", "metadata": {"name": "sre"}, "spec": {"replicas": 1}}) + reg = h.create({"kind": "KarsSRERegistration", "metadata": {"name": "canonical"}, "status": {}}) + assert_unissued(h, {"legacyBindings": []}, {}) + h.objects["karssreregistration", "canonical"]["status"]["routerServiceAccountUid"] = \ + h.state["prestaged_account"]["metadata"]["uid"] + with self.assertRaises(AssertionError): + assert_unissued(h, {"legacyBindings": []}, {}) + h.objects["karssreregistration", "canonical"] = reg + h.objects["serviceaccount", "sre-api-router"]["metadata"]["annotations"] = {OWNER: reg["metadata"]["uid"]} + with self.assertRaises(AssertionError): + assert_unissued(h, {"legacyBindings": []}, {}) + + def test_trusted_private_identity_requires_cleanup_new_uid_and_registration_owner(self): + h = self.seeded() + reg = {"metadata": {"uid": "registration"}, "status": {}} + with self.assertRaises(AssertionError): + assert_fresh_private_identity(h, reg) + h.state["prestaged_account_cleaned"] = True + del h.objects["secret", TOKEN_ALIAS] + original = h.state["prestaged_account"] + fresh = h.create({"kind": "ServiceAccount", "metadata": { + "name": "sre-api-router", "namespace": RUNTIME, "annotations": {OWNER: "registration"}}, + "automountServiceAccountToken": False}) + reg["status"]["routerServiceAccountUid"] = fresh["metadata"]["uid"] + assert_fresh_private_identity(h, reg) + for uid in (original["metadata"]["uid"], "unregistered"): + h.objects["serviceaccount", "sre-api-router"]["metadata"]["uid"] = uid + with self.assertRaises(AssertionError): + assert_fresh_private_identity(h, reg) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/migration.py b/tests/e2e/sre_authority/migration.py index 9b0a2a342..8e39c8882 100644 --- a/tests/e2e/sre_authority/migration.py +++ b/tests/e2e/sre_authority/migration.py @@ -9,7 +9,9 @@ ) from .fixtures import CONTROL, CONTROL_NS, GROUP_BINDING, delegate_operators from .admission import admission_cases, policies_ready -from .credential_paths import block_prestaged_paths, fixture_review, secret_watch, token_secret_denials +from .credential_paths import ( + assert_fresh_private_identity, block_prestaged_paths, fixture_review, secret_watch, token_secret_denials, +) def cli_failure(result, text, label): @@ -121,6 +123,8 @@ def migrated(): return reg if status.get("phase") == "Ready" and status.get("observedGeneration") == reg["metadata"]["generation"] else False reg = h.poll("reviewed migration and credential-order proof", migrated, seconds=240, interval=0.25) require(issuance_seen, "No private issuance was observed") + assert_fresh_private_identity(h, reg) + h.passed("Controller-created private ServiceAccount has a new registered UID, distinct from the operator-deleted unowned same-name fixture") h.cli("authority", "migrate", user="registrar", timeout=45) h.wait_ready() h.passed("Real old-principal GET/LIST/WATCH denial and old Pod termination precede private credential issuance") From ae692a333c6d673dfced5d8daa9485d76520605a Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 22:27:55 +0200 Subject: [PATCH 18/62] test(sre): retain sanitized migration rejection coordinates The real token fixture lifecycle now passes on hosted Kind, but a later reviewed migration is Blocked. Identify exact checked-in controller rejection sites and numeric Kubernetes statuses before teardown without printing status detail, API bodies or credentials; keep the fatal acceptance gate unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/common.py | 31 +++++++++++++++++++++++++ tests/e2e/sre_authority/harness_test.py | 25 +++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 547d75c60..8deea8ff0 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -80,6 +80,35 @@ def command_error_category(stderr): return "unclassified" +def authority_failure_site(root, detail): + """Report checked-in rejection coordinates, never the status detail itself.""" + if not isinstance(detail, str) or not detail: + return {"category": "unclassified"} + result = {"category": "controller-rejection"} + message = detail + status = re.fullmatch(r"(.+): Kubernetes status ([1-5][0-9]{2})", detail) + if status: + message = status.group(1) + result = {"category": "kubernetes-status", "httpStatus": int(status.group(2))} + elif detail.endswith(": Kubernetes transport/serialization failure"): + message = detail.removesuffix(": Kubernetes transport/serialization failure") + result = {"category": "kubernetes-transport"} + # Only an exact literal in our own production source can identify a site. + # Untrusted API messages, names, credentials and interpolated suffixes are + # not echoed, even if they contain a familiar rejection substring. + literal = json.dumps(message, ensure_ascii=False) + paths = [root / "controller/src/sre_authority.rs", root / "controller/src/sre_registration.rs", + root / "shared/sre_privacy.rs"] + paths += sorted((root / "controller/src/sre_authority").glob("*.rs")) + for path in paths: + if path.name.endswith("tests.rs"): + continue + for number, line in enumerate(path.read_text().splitlines(), 1): + if literal in line: + return {**result, "source": str(path.relative_to(root)), "line": number} + return {"category": "unclassified"} + + def printed_object(output): start = output.find("{") require(start >= 0, "CLI omitted its JSON object") @@ -356,6 +385,8 @@ def diagnostics(self): "observedGeneration": status.get("observedGeneration"), "generation": obj["metadata"].get("generation"), "availableReplicas": status.get("availableReplicas"), + **({"authorityFailure": authority_failure_site(self.root, status.get("detail"))} + if kind == "karssreregistrations.kars.azure.com" and status.get("phase") == "Blocked" else {}), "conditions": [{"type": condition.get("type"), "status": condition.get("status"), "reason": condition.get("reason")} for condition in status.get("conditions", [])]}), flush=True) diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 926b8f8d5..0380450e9 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -16,7 +16,7 @@ from sre_authority.common import ( CLAIM_VERSION, NAMESPACE_UID, RUNTIME, SOURCE_NAME, SOURCE_NS, SOURCE_UID, SYSTEM, - Harness, assert_claim, assert_denial, enrollment_json, printed_object, review_args, + Harness, assert_claim, assert_denial, authority_failure_site, enrollment_json, printed_object, review_args, ) from sre_authority.fixtures import seed_control_consumer from sre_authority.admission import reserved_source_probe @@ -34,6 +34,29 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_blocked_authority_diagnostics_report_source_coordinates_not_status_detail(self): + root = Path(__file__).resolve().parents[3] + cases = [ + ("Reviewed SRE consumer was changed or replaced", "migration.rs", "controller-rejection", None), + ("Private SRE role definition has excessive or unsupported authority", "bindings.rs", "controller-rejection", None), + ("Retire reviewed SRE ClusterRoleBinding: Kubernetes status 403", "bindings.rs", "kubernetes-status", 403), + ("Inspect reserved SRE token identity: Kubernetes transport/serialization failure", + "credential_guard.rs", "kubernetes-transport", None), + ] + for detail, source, category, code in cases: + with self.subTest(source=source): + result = authority_failure_site(root, detail) + self.assertTrue(result["source"].endswith(source)) + self.assertGreater(result["line"], 0) + self.assertEqual(result["category"], category) + self.assertEqual(result.get("httpStatus"), code) + self.assertNotIn(detail, json.dumps(result)) + for unsafe in (detail + MARKER, MARKER + detail, detail + "\n" + MARKER): + self.assertEqual(authority_failure_site(root, unsafe), {"category": "unclassified"}) + for unsafe in (None, {}, "", MARKER, "private body: Kubernetes status 403", + "Retire reviewed SRE ClusterRoleBinding: Kubernetes status 403 " + MARKER): + self.assertEqual(authority_failure_site(root, unsafe), {"category": "unclassified"}) + def test_control_consumer_fixture_satisfies_required_sandbox_fields(self): captured = [] class Captured(Exception): From 59f25ac360398a3cc5d91744b8e9d88f043e8a96 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 22:59:10 +0200 Subject: [PATCH 19/62] test(sre): preserve token fixture parents after cleanup conflicts Do not delete a fixture ServiceAccount if Secret cleanup fails: native TokenController could otherwise garbage-collect a foreign token Secret replacement. Independent cleanup continues and the original acceptance failure remains primary. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/credential_paths.py | 4 ++++ tests/e2e/sre_authority/credential_paths_test.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/e2e/sre_authority/credential_paths.py b/tests/e2e/sre_authority/credential_paths.py index bfe51596a..cdc5c6585 100644 --- a/tests/e2e/sre_authority/credential_paths.py +++ b/tests/e2e/sre_authority/credential_paths.py @@ -111,6 +111,10 @@ def owned_fixtures(h): finally: failed = False for path, obj in reversed(fixtures): + # Deleting the SA after a Secret CAS conflict could let native + # TokenController garbage-collect the foreign replacement. + if failed and obj.get("kind") == "ServiceAccount": + continue try: delete_owned(h, path, obj) except Exception: diff --git a/tests/e2e/sre_authority/credential_paths_test.py b/tests/e2e/sre_authority/credential_paths_test.py index e70949373..ab50501aa 100644 --- a/tests/e2e/sre_authority/credential_paths_test.py +++ b/tests/e2e/sre_authority/credential_paths_test.py @@ -178,7 +178,7 @@ def test_cleanup_accepts_only_real_absence_and_never_drops_cas_fences(self): with patch.object(h, "api", return_value=Response(404, {})), self.assertRaises(AssertionError): delete_owned(h, path, obj) - def test_cleanup_conflict_cannot_mask_primary_error_and_all_fixtures_are_attempted(self): + def test_cleanup_conflict_preserves_primary_error_and_attempts_independent_fixtures(self): h = self.seeded() output = io.StringIO() with patch("sre_authority.credential_paths.delete_owned", side_effect=AssertionError("private-body")) as cleanup: @@ -192,6 +192,19 @@ def test_cleanup_conflict_cannot_mask_primary_error_and_all_fixtures_are_attempt with owned_fixtures(h) as fixtures: fixtures.append(("first", {})) + def test_secret_conflict_preserves_account_to_prevent_gc_of_foreign_replacement(self): + h = self.seeded() + original = h.state["prestaged_alias"] + replacement = h.create(original) + account = h.state["prestaged_account"] + with self.assertRaisesRegex(AssertionError, "Fixture cleanup failed"): + with owned_fixtures(h) as fixtures: + fixtures.append((f"/api/v1/namespaces/{RUNTIME}/serviceaccounts/sre-api-router", account)) + fixtures.append((SECRET_PATH + "/" + TOKEN_ALIAS, original)) + self.assertEqual(h.get("secret", TOKEN_ALIAS), replacement) + self.assertEqual(h.get("serviceaccount", "sre-api-router"), account) + self.assertEqual(len(h.requests), 1) + def test_disappeared_fixture_patch_404_is_fatal_even_when_cleanup_is_404(self): h = self.seeded() original = h.api From fa09d3b722ffe464b48eedf555ac8db93a13c36d Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 23:23:14 +0200 Subject: [PATCH 20/62] test(sre): prove controller retirement bind authority on real Kind Run a bearer-only actual controller ServiceAccount UID proof against the unchanged chart policies and roles, using pinned historical reader bindings with unrelated survivors. Compare exact UID/RV-fenced retirement dry-runs before, during and after a disposable exact-named reader bind grant. Preserve custom-role denials, pending-only admission and all persistent reviewed resources; never execute images or publish credentials. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 4 +- tests/e2e/sre_authority/binding_probe.py | 322 ++++++++++++++++++ tests/e2e/sre_authority/binding_probe_test.py | 109 ++++++ tests/e2e/sre_authority/bootstrap_probe.py | 21 +- 4 files changed, 450 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/sre_authority/binding_probe.py create mode 100644 tests/e2e/sre_authority/binding_probe_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b4fe2434..872c4291b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,7 +416,7 @@ jobs: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Check public-schema diagnostic privacy - run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test + run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test - name: Create the same disposable API server as the real harness run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" - name: Validate the SRE CRD against the actual API server @@ -424,7 +424,7 @@ jobs: run: python3 tests/e2e/sre_authority/registration_schema.py --exercise - name: Prove controller Pod admission with all chart policies and no image execution id: sre_bootstrap - run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe + run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --retirement-bind-proof - name: Collect nil-schema adapter candidate evidence without weakening production gates if: failure() && steps.sre_bootstrap.outcome == 'failure' run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --json-params-candidate diff --git a/tests/e2e/sre_authority/binding_probe.py b/tests/e2e/sre_authority/binding_probe.py new file mode 100644 index 000000000..f39336b98 --- /dev/null +++ b/tests/e2e/sre_authority/binding_probe.py @@ -0,0 +1,322 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Image-free, real-controller-principal RBAC retirement dry-run experiment.""" + +import base64 +import json +import re +import ssl +import time +from urllib.error import HTTPError +from urllib.parse import urlsplit +from urllib.request import HTTPSHandler, ProxyHandler, Request, build_opener + +from .common import POLICIES, REGISTRATION, RUNTIME, SYSTEM, require +from .fixtures import LEGACY_COMMIT +from .registration_schema import CONTEXT, command, request + +RBAC = "/apis/rbac.authorization.k8s.io/v1" +READER = "kars-sre-reader" +CUSTOM = "e2e-sre-custom-review" +EXTRA = "e2e-sre-reader-bind-proof" +CONTROLLER = f"system:serviceaccount:{SYSTEM}:kars-controller" +RETIRED = "kars.azure.com/sre-legacy-retired" +SURVIVOR = {"kind": "ServiceAccount", "name": "e2e-retained-subject", "namespace": SYSTEM} +LEGACY = {"kind": "ServiceAccount", "name": "sandbox", "namespace": RUNTIME} + + +def create(port, path, obj): + code, result = request(port, "POST", path, obj) + require(code == 201 and result.get("metadata", {}).get("uid") + and result["metadata"].get("resourceVersion"), "RBAC proof fixture CREATE failed") + return result + + +def get(port, path): + code, obj = request(port, "GET", path) + require(code == 200 and isinstance(obj, dict), "RBAC proof fixture GET failed") + return obj + + +def historical_reader(root): + from .bootstrap_probe import converted_objects + command("reader-history", ["git", "fetch", "--no-tags", "--depth=1", + "https://github.com/Azure/kars.git", LEGACY_COMMIT], root=root) + chart = root / ".e2e-sre-reader-chart" + require(not chart.exists(), "Refusing to overwrite an existing historical reader fixture directory") + files = ("Chart.yaml", "values.yaml", "templates/_helpers.tpl", "templates/sre.yaml") + try: + for name in files: + contents = command("reader-source", ["git", "show", + f"{LEGACY_COMMIT}:deploy/helm/kars/{name}"], root=root) + path = chart / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents) + rendered = command("reader-render", ["helm", "template", "kars", str(chart), + "--namespace", SYSTEM, "--set", "sre.enabled=true", "--show-only", "templates/sre.yaml"], root=root) + documents = [document for document in re.split(r"(?m)^---\s*\n", rendered) + if re.search(r"(?m)^kind: ClusterRole\s*$", document) + and re.search(r"(?m)^ name: kars-sre-reader\s*$", document)] + require(len(documents) == 1, "Pinned legacy chart did not emit exactly one reader role") + objects = converted_objects(command("reader-convert", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=documents[0])) + require(len(objects) == 1 and objects[0]["metadata"]["name"] == READER + and objects[0]["kind"] == "ClusterRole", "Unexpected historical RBAC fixture") + return objects[0] + finally: + for name in files: + (chart / name).unlink(missing_ok=True) + for path in (chart / "templates", chart): + if path.exists(): + path.rmdir() + + +def seed(port, root): + code, _ = request(port, "GET", "/apis/admissionregistration.k8s.io/v1/" + "validatingadmissionpolicies/kars-sre-binding-authority") + require(code == 404, "Legacy binding proof fixtures must precede admission policies") + reader = create(port, RBAC + "/clusterroles", historical_reader(root)) + custom = create(port, RBAC + "/clusterroles", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", + "metadata": {"name": CUSTOM}, + "rules": [{"apiGroups": [""], "resources": ["limitranges"], "verbs": ["get"]}]}) + bindings = [] + for role in (reader, custom): + bindings.append(create(port, RBAC + "/clusterrolebindings", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", + "metadata": {"name": role["metadata"]["name"]}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", + "name": role["metadata"]["name"]}, + "subjects": [LEGACY, SURVIVOR]})) + consumer = create(port, f"/apis/apps/v1/namespaces/{RUNTIME}/deployments", { + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "sre", "namespace": RUNTIME}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-reviewed-consumer"}}, + "template": {"metadata": {"labels": {"app": "e2e-reviewed-consumer"}}, + "spec": {"automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "never-executed", + "image": "registry.invalid/kars-admission-proof:never", "imagePullPolicy": "Never"}]}}}}) + return {"bindings": bindings, "roles": [reader, custom], "consumer": consumer} + + +def retirement_patch(binding, registration_uid): + require(binding["subjects"] == [LEGACY, SURVIVOR], "Unexpected reviewed binding subjects") + return {"metadata": {"uid": binding["metadata"]["uid"], + "resourceVersion": binding["metadata"]["resourceVersion"], + "annotations": {RETIRED: registration_uid}}, "subjects": [SURVIVOR]} + + +def authorization_category(code, body): + if not isinstance(body, dict): + return "unexpected-response" + if code == 403 and body.get("kind") == "Status" and body.get("reason") == "Forbidden": + message = body.get("message", "") + if isinstance(message, str) and "is attempting to grant RBAC permissions not currently held" in message: + return "rbac-permissions-not-held" + return "other-forbidden" + if code == 409 and body.get("kind") == "Status" and body.get("reason") == "Conflict": + return "cas-conflict" + if code == 200 and body.get("kind") == "ClusterRoleBinding": + return "accepted" + return "unexpected-response" + + +class ControllerAPI: + def __init__(self, root, port): + config_args = ["kubectl", "--context", CONTEXT, "config", "view", "--raw", "--minify", "-o"] + self.server = command("public-api-server", config_args + [ + "jsonpath={.clusters[0].cluster.server}"], root=root) + require(urlsplit(self.server).hostname in ("127.0.0.1", "localhost", "::1"), + "RBAC proof refuses a non-loopback API") + ca = command("public-api-ca", config_args + [ + "jsonpath={.clusters[0].cluster.certificate-authority-data}"], root=root) + context = ssl.create_default_context( + cadata=base64.b64decode(ca).decode()) + self.opener = build_opener(ProxyHandler({}), HTTPSHandler(context=context)) + account = get(port, f"/api/v1/namespaces/{SYSTEM}/serviceaccounts/kars-controller") + code, response = request(port, "POST", f"/api/v1/namespaces/{SYSTEM}/serviceaccounts/kars-controller/token", { + "apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest", + "spec": {"audiences": [], "expirationSeconds": 600}}) + require(code == 201 and response.get("status", {}).get("token"), + "Ephemeral controller-principal TokenRequest failed") + self.token = response["status"]["token"] + code, response = self.request("POST", "/apis/authentication.k8s.io/v1/selfsubjectreviews", { + "apiVersion": "authentication.k8s.io/v1", "kind": "SelfSubjectReview"}) + identity = response.get("status", {}).get("userInfo", {}) + require(code == 201 and identity.get("username") == CONTROLLER + and identity.get("uid") == account["metadata"]["uid"], + "Bearer-only proof principal does not match the actual controller ServiceAccount UID") + self.account = account + + def request(self, method, path, obj): + req = Request(self.server + path, data=json.dumps(obj).encode(), method=method, + headers={"Content-Type": "application/merge-patch+json" if method == "PATCH" else "application/json", + "Accept": "application/json", "Authorization": f"Bearer {self.token}"}) + try: + response = self.opener.open(req, timeout=15) + except HTTPError as error: + response = error + with response: + return response.code, json.loads(response.read(1024 * 1024)) + + def allowed(self, group, resource, verb, name=None): + attributes = {"group": group, "resource": resource, "verb": verb} + if name: + attributes["name"] = name + code, result = self.request("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", { + "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", + "spec": {"resourceAttributes": attributes}}) + require(code == 201 and isinstance(result.get("status", {}).get("allowed"), bool) + and not result["status"].get("evaluationError"), "Live controller authorization review failed") + return result["status"]["allowed"] + + +def observe_bind(api, expected): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", READER) is expected: + return + time.sleep(0.25) + raise AssertionError("Exact named bind permission did not converge") + + +def prove(root, port, state, objects, report): + from .bootstrap_probe import PATHS + source = create(port, f"/apis/kars.azure.com/v1alpha1/namespaces/{SYSTEM}/karssandboxes", { + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSandbox", + "metadata": {"name": "sre", "namespace": SYSTEM}, + "spec": {"runtime": {"kind": "BYO", "byo": {"contractVersion": "v1", + "image": "registry.invalid/kars-admission-proof:never", "command": ["/bin/true"]}}, + "inferenceRef": {"name": "sre-inference"}, "sandbox": {"isolation": "standard"}}}) + consumer_path = f"/apis/apps/v1/namespaces/{RUNTIME}/deployments/sre" + consumer = get(port, consumer_path) + spec = {"controller": {"namespace": {"name": SYSTEM, + "uid": get(port, f"/api/v1/namespaces/{SYSTEM}")["metadata"]["uid"]}, + "deployment": {"name": "kars-controller", "uid": get(port, + f"/apis/apps/v1/namespaces/{SYSTEM}/deployments/kars-controller")["metadata"]["uid"]}, "release": "kars"}, + "sandbox": {"namespace": SYSTEM, "name": "sre", "uid": source["metadata"]["uid"]}, + "runtimeNamespace": {"name": RUNTIME, "uid": get(port, f"/api/v1/namespaces/{RUNTIME}")["metadata"]["uid"]}, + "legacyBindings": [{"kind": "ClusterRoleBinding", **{k: binding["metadata"][k] + for k in ("name", "uid", "resourceVersion")}, "roleRef": binding["roleRef"], "subjects": binding["subjects"]} + for binding in state["bindings"]], + "legacyConsumer": {"namespace": RUNTIME, "name": "sre", **{k: consumer["metadata"][k] + for k in ("uid", "resourceVersion")}}} + registration = create(port, REGISTRATION.rsplit("/", 1)[0], { + "apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", + "metadata": {"name": "canonical"}, "spec": spec}) + snapshots = {} + for obj in objects + state["roles"] + state["bindings"]: + if obj["kind"] not in ("ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding", + "ValidatingAdmissionPolicy", "ValidatingAdmissionPolicyBinding"): + continue + path = PATHS[obj["kind"]].format(namespace=obj["metadata"].get("namespace", SYSTEM)) + path += "/" + obj["metadata"]["name"] + current = get(port, path) + if obj["kind"] == "ValidatingAdmissionPolicy" and obj["metadata"]["name"].startswith("kars-sre-"): + require(current.get("status", {}).get("observedGeneration") == current["metadata"]["generation"] + and "typeChecking" in current["status"] + and not current["status"]["typeChecking"].get("expressionWarnings"), + "SRE policy is not observed and warning-free during RBAC proof") + if obj["kind"] == "ValidatingAdmissionPolicyBinding" and obj["metadata"]["name"].startswith("kars-sre-"): + require("Deny" in current["spec"]["validationActions"] + and current["spec"]["policyName"] == current["metadata"]["name"], + "SRE policy lacks its intended Deny binding during RBAC proof") + # Native CEL type-check status may evolve; specification/identity must not. + current.pop("status", None) + if obj["kind"] == "ValidatingAdmissionPolicy": + current["metadata"].pop("resourceVersion", None) + snapshots[path] = current + require({obj["metadata"]["name"] for obj in objects if obj["kind"] == "ValidatingAdmissionPolicy" + and obj["metadata"]["name"].startswith("kars-sre-")} == {"kars-sre-" + name for name in POLICIES}, + "RBAC experiment lacks the complete SRE policy set") + api = ControllerAPI(root, port) + facts = {"controllerBearerUidVerified": True, "workloadExecution": "not-attempted", + "legacyCommit": LEGACY_COMMIT, "observedWarningFreeDenyBoundSrePolicies": len(POLICIES), "cases": []} + report(facts) + require(api.allowed("rbac.authorization.k8s.io", "clusterrolebindings", "patch", READER), + "Controller lacks ordinary CRB PATCH; this is not the bind hypothesis") + require(api.allowed("kars.azure.com", "karssreregistrations", "use", "canonical"), + "Controller lacks canonical registrar use") + baseline_bind = api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", READER) + facts["authorization"] = {"patchReaderBinding": True, "useCanonical": True, "bindReader": baseline_bind, + "bindCustom": api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", CUSTOM), + "getCustomLimitRanges": api.allowed("", "limitranges", "get"), + "getReaderNodeMetrics": api.allowed("metrics.k8s.io", "nodes", "get")} + require(not facts["authorization"]["bindCustom"] and not facts["authorization"]["getCustomLimitRanges"], + "Custom fixture is not outside the controller's current authority") + + def unchanged(): + for path, before in snapshots.items(): + current = get(port, path) + current.pop("status", None) + if current["kind"] == "ValidatingAdmissionPolicy": + current["metadata"].pop("resourceVersion", None) + require(current == before, "RBAC experiment changed a policy, role or reviewed binding") + live = get(port, consumer_path) + require(all(live["metadata"][key] == consumer["metadata"][key] for key in ("uid", "resourceVersion")) + and live["spec"] == consumer["spec"], + "RBAC experiment changed or stopped the legacy consumer") + require(get(port, REGISTRATION) == registration, "Image-free experiment unexpectedly reconciled registration") + require(get(port, f"/api/v1/namespaces/{SYSTEM}/serviceaccounts/kars-controller") == api.account, + "Controller identity was replaced during the RBAC experiment") + for secret in ("sre-api-router-identity", "sre-api-agent"): + code, _ = request(port, "GET", f"/api/v1/namespaces/{RUNTIME}/secrets/{secret}") + require(code == 404, "RBAC dry-run experiment issued private material") + + def probe(binding, phase, expected, stale=None): + patch = retirement_patch(binding, registration["metadata"]["uid"]) + if stale: + patch["metadata"][stale] = "00000000-0000-0000-0000-000000000000" if stale == "uid" else "1" + code, response = api.request("PATCH", + RBAC + "/clusterrolebindings/" + binding["metadata"]["name"] + "?dryRun=All", patch) + category = authorization_category(code, response) + result = {"phase": phase, "role": binding["roleRef"]["name"], "httpStatus": code, "category": category} + facts["cases"].append(result) + report(facts) + require(code == expected and category == {200: "accepted", 403: "rbac-permissions-not-held", + 409: "cas-conflict"}[expected], "Retirement dry-run did not prove the intended API outcome") + if expected == 200: + require(response["metadata"]["uid"] == binding["metadata"]["uid"] + and response["metadata"].get("annotations", {}).get(RETIRED) == registration["metadata"]["uid"] + and response["roleRef"] == binding["roleRef"] and response["subjects"] == [SURVIVOR], + "Retirement dry-run did not preserve the exact unrelated subject and roleRef") + unchanged() + + reader, custom = state["bindings"] + probe(reader, "baseline", 200 if baseline_bind else 403) + probe(custom, "baseline-custom", 403) + extra = [] + try: + role = create(port, RBAC + "/clusterroles", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRole", "metadata": {"name": EXTRA}, + "rules": [{"apiGroups": ["rbac.authorization.k8s.io"], "resources": ["clusterroles"], + "resourceNames": [READER], "verbs": ["bind"]}]}) + extra.append((RBAC + "/clusterroles/" + EXTRA, role)) + binding = create(port, RBAC + "/clusterrolebindings", { + "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "ClusterRoleBinding", "metadata": {"name": EXTRA}, + "roleRef": {"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": EXTRA}, + "subjects": [{"kind": "ServiceAccount", "name": "kars-controller", "namespace": SYSTEM}]}) + extra.append((RBAC + "/clusterrolebindings/" + EXTRA, binding)) + observe_bind(api, True) + require(not api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", CUSTOM) + and not api.allowed("", "limitranges", "get"), "Named reader bind widened custom-role authority") + probe(reader, "exact-reader-bind-only", 200) + probe(custom, "exact-reader-bind-only-custom", 403) + probe(reader, "wrong-uid", 409, "uid") + probe(reader, "stale-resource-version", 409, "resourceVersion") + facts["allReviewedDryRunsAuthorized"] = False + facts["noRetirementsApplied"] = True + finally: + for path, obj in reversed(extra): + code, _ = request(port, "DELETE", path, { + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {k: obj["metadata"][k] for k in ("uid", "resourceVersion")}}) + require(code in (200, 202), "Exact named bind fixture cleanup failed") + observe_bind(api, baseline_bind) + probe(reader, "named-bind-removed", 200 if baseline_bind else 403) + facts["unchangedPoliciesRolesBindingsConsumer"] = True + facts["testBindGrantRemoved"] = True + report(facts) diff --git a/tests/e2e/sre_authority/binding_probe_test.py b/tests/e2e/sre_authority/binding_probe_test.py new file mode 100644 index 000000000..dd5431988 --- /dev/null +++ b/tests/e2e/sre_authority/binding_probe_test.py @@ -0,0 +1,109 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure privacy/request-shape checks; the A/B result requires actual Kind.""" + +import base64 +import copy +import json +from pathlib import Path +import types +import unittest +from unittest.mock import patch + +from sre_authority.binding_probe import ( + CONTROLLER, LEGACY, RETIRED, SURVIVOR, ControllerAPI, authorization_category, retirement_patch, +) + + +class BindingProbeTests(unittest.TestCase): + def test_patch_is_exact_subtractive_review_with_uid_rv_and_registration_fence(self): + binding = {"metadata": {"uid": "binding-uid", "resourceVersion": "123"}, + "subjects": [LEGACY, SURVIVOR]} + original = copy.deepcopy(binding) + result = retirement_patch(binding, "registration-uid") + self.assertEqual(result, {"metadata": {"uid": "binding-uid", "resourceVersion": "123", + "annotations": {RETIRED: "registration-uid"}}, "subjects": [SURVIVOR]}) + self.assertEqual(binding, original) + for subjects in ([SURVIVOR], [LEGACY], [LEGACY, SURVIVOR, {"kind": "User", "name": "unreviewed"}]): + with self.assertRaises(AssertionError): + retirement_patch({**binding, "subjects": subjects}, "registration-uid") + + def test_arbitrary_forbidden_or_failure_is_not_escalation_proof(self): + denied = {"kind": "Status", "reason": "Forbidden", + "message": "user is attempting to grant RBAC permissions not currently held: private-not-for-logs"} + self.assertEqual(authorization_category(403, denied), "rbac-permissions-not-held") + for message in ("cannot patch clusterrolebindings", "kars-sre-binding-authority denied", None, {}, ""): + self.assertEqual(authorization_category(403, {**denied, "message": message}), "other-forbidden") + for code in (200, 201, 401, 404, 409, 422, 500): + self.assertEqual(authorization_category(code, denied), "unexpected-response") + self.assertEqual(authorization_category(200, {"kind": "ClusterRoleBinding"}), "accepted") + self.assertEqual(authorization_category(409, {"kind": "Status", "reason": "Conflict"}), "cas-conflict") + self.assertEqual(authorization_category(403, None), "unexpected-response") + self.assertNotIn("private-not-for-logs", authorization_category(403, denied)) + + def test_controller_token_is_memory_only_no_admin_certificate_and_real_uid_is_required(self): + account = {"metadata": {"uid": "actual-controller-uid"}} + captured = [] + def api_request(_self, method, path, obj): + captured.append((method, path, obj)) + return 201, {"status": {"userInfo": {"username": CONTROLLER, "uid": "actual-controller-uid"}}} + def config(stage, *_args, **_kwargs): + return "https://127.0.0.1:6443" if stage == "public-api-server" else base64.b64encode(b"public CA").decode() + context = types.SimpleNamespace() + with patch("sre_authority.binding_probe.command", side_effect=config), \ + patch("sre_authority.binding_probe.get", return_value=account), \ + patch("sre_authority.binding_probe.request", return_value=(201, {"status": {"token": "unit-only-token"}})), \ + patch("sre_authority.binding_probe.ssl.create_default_context", return_value=context) as ssl_context, \ + patch("sre_authority.binding_probe.HTTPSHandler") as https, \ + patch("sre_authority.binding_probe.build_opener"), \ + patch.object(ControllerAPI, "request", api_request): + client = ControllerAPI(Path("."), 1) + self.assertEqual(client.account, account) + ssl_context.assert_called_once_with(cadata="public CA") + https.assert_called_once_with(context=context) + self.assertTrue(captured[0][1].endswith("/selfsubjectreviews")) + with patch.object(ControllerAPI, "request", return_value=(201, { + "status": {"userInfo": {"username": CONTROLLER, "uid": "replacement"}}})): + with self.assertRaises(AssertionError): + ControllerAPI(Path("."), 1) + + def test_controller_dry_run_has_real_bearer_and_merge_patch_without_impersonation(self): + class Response: + code = 403 + def __enter__(self): + return self + def __exit__(self, *_args): + pass + def read(self, _limit): + return b'{"kind":"Status","reason":"Forbidden"}' + sent = [] + def open_request(request, **_kwargs): + sent.append(request) + return Response() + client = ControllerAPI.__new__(ControllerAPI) + client.server, client.token = "https://127.0.0.1:6443", "unit-only-token" + client.opener = types.SimpleNamespace(open=open_request) + code, _ = client.request("PATCH", "/fixture?dryRun=All", {"subjects": [SURVIVOR]}) + self.assertEqual(code, 403) + self.assertEqual(sent[0].get_header("Authorization"), "Bearer unit-only-token") + self.assertEqual(sent[0].get_header("Content-type"), "application/merge-patch+json") + self.assertFalse(any(key.lower().startswith("impersonate") for key in sent[0].headers)) + self.assertEqual(json.loads(sent[0].data), {"subjects": [SURVIVOR]}) + + def test_fast_hosted_gate_keeps_prior_admission_cases_and_adds_real_bind_experiment(self): + root = Path(__file__).resolve().parents[3] + workflow = (root / ".github/workflows/ci.yml").read_text() + fast = workflow.split(" sre-crd-schema:", 1)[1].split(" helm-lint:", 1)[0] + self.assertIn("sre_authority.bootstrap_probe --retirement-bind-proof", fast) + self.assertIn("sre_authority.binding_probe_test", fast) + source = (root / "tests/e2e/sre_authority/bootstrap_probe.py").read_text() + self.assertLess(source.index("cases = admission_cases"), source.index("prove(root, port, state")) + probe = (root / "tests/e2e/sre_authority/binding_probe.py").read_text() + self.assertIn('"resourceNames": [READER], "verbs": ["bind"]', probe) + self.assertNotIn('"verbs": ["escalate"]', probe) + self.assertNotIn('"resourceNames": ["*"]', probe) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index bc8da60da..d7268cd2d 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -125,13 +125,17 @@ def upsert(port, obj, policies): return result -def exercise(root, port, objects, policies, wait_seconds=90): +def exercise(root, port, objects, policies, wait_seconds=90, retirement=False): results = [] for name in ("kars-system", "kars-sre"): code, body = request(port, "POST", "/api/v1/namespaces", { "apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}}) if code not in (201, 409): raise RuntimeError("Disposable bootstrap namespace unavailable") + retirement_state = None + if retirement: + from sre_authority.binding_probe import seed + retirement_state = seed(port, root) for kind in PATHS: if kind == "Deployment": continue @@ -185,9 +189,10 @@ def exercise(root, port, objects, policies, wait_seconds=90): "allPoliciesObservedBeforeCreate": observed}) if not observed: raise RuntimeError("Public admission policy observation timed out; Pod evidence was still collected") + return retirement_state -def main(root, diagnostics_only, candidate=False): +def main(root, diagnostics_only, candidate=False, retirement=False): global REPORT_PREFIX REPORT_PREFIX = "candidate-" if candidate else "" with kind_proxy(root) as (port, version): @@ -205,12 +210,17 @@ def main(root, diagnostics_only, candidate=False): CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) else: try: - exercise(root, port, objects, policies, wait_seconds=180 if candidate else 90) + state = exercise(root, port, objects, policies, wait_seconds=180 if candidate else 90, + retirement=retirement and not candidate) from sre_authority.bootstrap_cases import admission_cases cases = admission_cases(port, policies) write_report(root, "bootstrap-admission-cases.json", {"cases": cases}) if not all(case["matched"] for case in cases): raise RuntimeError("Schema failed intended ordinary/private admission outcomes") + if state: + from sre_authority.binding_probe import prove + prove(root, port, state, objects, + lambda facts: write_report(root, "bootstrap-binding-retirement.json", facts)) finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( @@ -223,9 +233,12 @@ def main(root, diagnostics_only, candidate=False): parser.add_argument("--diagnostics-only", action="store_true") parser.add_argument("--json-params-candidate", action="store_true", help="Diagnose only the API-evidenced nil-schema adapter candidate; never edit production") + parser.add_argument("--retirement-bind-proof", action="store_true", + help="Prove controller-principal retirement dry-runs with a test-only exact reader bind grant") args = parser.parse_args() try: - main(Path(__file__).resolve().parents[3], args.diagnostics_only, args.json_params_candidate) + main(Path(__file__).resolve().parents[3], args.diagnostics_only, args.json_params_candidate, + args.retirement_bind_proof) except Exception as error: print(f"SRE-BOOTSTRAP-FAIL category={type(error).__name__}", flush=True) raise SystemExit(1) from None From b935015f068f7a296fb72a3a29f26b15d8927ebc Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 23:26:52 +0200 Subject: [PATCH 21/62] test(sre): use the exact pinned legacy chart file set The hosted proof found that the immutable legacy chart has no helpers template. Render only its three actual required files, and report sanitized public diagnostic source coordinates without exception text or request data. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/binding_probe.py | 2 +- tests/e2e/sre_authority/binding_probe_test.py | 11 +++++++++++ tests/e2e/sre_authority/bootstrap_probe.py | 13 +++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/e2e/sre_authority/binding_probe.py b/tests/e2e/sre_authority/binding_probe.py index f39336b98..378a616c4 100644 --- a/tests/e2e/sre_authority/binding_probe.py +++ b/tests/e2e/sre_authority/binding_probe.py @@ -45,7 +45,7 @@ def historical_reader(root): "https://github.com/Azure/kars.git", LEGACY_COMMIT], root=root) chart = root / ".e2e-sre-reader-chart" require(not chart.exists(), "Refusing to overwrite an existing historical reader fixture directory") - files = ("Chart.yaml", "values.yaml", "templates/_helpers.tpl", "templates/sre.yaml") + files = ("Chart.yaml", "values.yaml", "templates/sre.yaml") try: for name in files: contents = command("reader-source", ["git", "show", diff --git a/tests/e2e/sre_authority/binding_probe_test.py b/tests/e2e/sre_authority/binding_probe_test.py index dd5431988..05bc3bd44 100644 --- a/tests/e2e/sre_authority/binding_probe_test.py +++ b/tests/e2e/sre_authority/binding_probe_test.py @@ -14,9 +14,20 @@ from sre_authority.binding_probe import ( CONTROLLER, LEGACY, RETIRED, SURVIVOR, ControllerAPI, authorization_category, retirement_patch, ) +from sre_authority.bootstrap_probe import failure_site class BindingProbeTests(unittest.TestCase): + def test_proof_failure_reports_only_public_source_coordinate_not_values(self): + try: + retirement_patch({"subjects": ["private-body-do-not-log"]}, "private-token-do-not-log") + except AssertionError as error: + facts = failure_site(error) + self.assertEqual(facts["category"], "AssertionError") + self.assertEqual(facts["source"], "binding_probe.py") + self.assertGreater(facts["line"], 0) + self.assertNotIn("private", json.dumps(facts)) + def test_patch_is_exact_subtractive_review_with_uid_rv_and_registration_fence(self): binding = {"metadata": {"uid": "binding-uid", "resourceVersion": "123"}, "subjects": [LEGACY, SURVIVOR]} diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index d7268cd2d..22cedbfcb 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -20,6 +20,18 @@ def write_report(root, filename, report): schema_report(root, REPORT_PREFIX + filename, report) + +def failure_site(error): + result = {"category": type(error).__name__} + frame = error.__traceback__ + while frame: + name = Path(frame.tb_frame.f_code.co_filename).name + if name in ("bootstrap_probe.py", "binding_probe.py"): + result.update(source=name, line=frame.tb_lineno) + frame = frame.tb_next + return result + + PATHS = { "CustomResourceDefinition": "/apis/apiextensions.k8s.io/v1/customresourcedefinitions", "ServiceAccount": "/api/v1/namespaces/{namespace}/serviceaccounts", @@ -241,4 +253,5 @@ def main(root, diagnostics_only, candidate=False, retirement=False): args.retirement_bind_proof) except Exception as error: print(f"SRE-BOOTSTRAP-FAIL category={type(error).__name__}", flush=True) + write_report(Path(__file__).resolve().parents[3], "bootstrap-failure-site.json", failure_site(error)) raise SystemExit(1) from None From c1d14fad4bbaeeda6832d278943ba9bba4e9130e Mon Sep 17 00:00:00 2001 From: pallakatos Date: Tue, 8 Sep 2026 23:29:48 +0200 Subject: [PATCH 22/62] test(sre): distinguish the real immutable-UID retirement rejection Hosted Kind proved the same controller-principal retirement PATCH is denied by RBAC without named reader bind and accepted with only that bind, while the custom role remains denied. The wrong-UID control is Kubernetes 422 immutable-field validation, not resourceVersion Conflict; require that exact metadata.uid cause without counting arbitrary 422s as proof. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/binding_probe.py | 10 ++++++++-- tests/e2e/sre_authority/binding_probe_test.py | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/e2e/sre_authority/binding_probe.py b/tests/e2e/sre_authority/binding_probe.py index 378a616c4..3da49f575 100644 --- a/tests/e2e/sre_authority/binding_probe.py +++ b/tests/e2e/sre_authority/binding_probe.py @@ -119,6 +119,11 @@ def authorization_category(code, body): return "other-forbidden" if code == 409 and body.get("kind") == "Status" and body.get("reason") == "Conflict": return "cas-conflict" + if code == 422 and body.get("kind") == "Status" and body.get("reason") == "Invalid": + causes = body.get("details", {}).get("causes", []) + if any(cause.get("field") == "metadata.uid" and cause.get("reason") == "FieldValueInvalid" + and "field is immutable" in cause.get("message", "") for cause in causes): + return "immutable-uid" if code == 200 and body.get("kind") == "ClusterRoleBinding": return "accepted" return "unexpected-response" @@ -277,7 +282,8 @@ def probe(binding, phase, expected, stale=None): facts["cases"].append(result) report(facts) require(code == expected and category == {200: "accepted", 403: "rbac-permissions-not-held", - 409: "cas-conflict"}[expected], "Retirement dry-run did not prove the intended API outcome") + 409: "cas-conflict", 422: "immutable-uid"}[expected], + "Retirement dry-run did not prove the intended API outcome") if expected == 200: require(response["metadata"]["uid"] == binding["metadata"]["uid"] and response["metadata"].get("annotations", {}).get(RETIRED) == registration["metadata"]["uid"] @@ -305,7 +311,7 @@ def probe(binding, phase, expected, stale=None): and not api.allowed("", "limitranges", "get"), "Named reader bind widened custom-role authority") probe(reader, "exact-reader-bind-only", 200) probe(custom, "exact-reader-bind-only-custom", 403) - probe(reader, "wrong-uid", 409, "uid") + probe(reader, "wrong-uid", 422, "uid") probe(reader, "stale-resource-version", 409, "resourceVersion") facts["allReviewedDryRunsAuthorized"] = False facts["noRetirementsApplied"] = True diff --git a/tests/e2e/sre_authority/binding_probe_test.py b/tests/e2e/sre_authority/binding_probe_test.py index 05bc3bd44..ff60cebfc 100644 --- a/tests/e2e/sre_authority/binding_probe_test.py +++ b/tests/e2e/sre_authority/binding_probe_test.py @@ -53,6 +53,15 @@ def test_arbitrary_forbidden_or_failure_is_not_escalation_proof(self): self.assertEqual(authorization_category(403, None), "unexpected-response") self.assertNotIn("private-not-for-logs", authorization_category(403, denied)) + def test_uid_rejection_is_specific_builtin_validation_not_arbitrary_422(self): + cause = {"field": "metadata.uid", "reason": "FieldValueInvalid", "message": "field is immutable"} + response = {"kind": "Status", "reason": "Invalid", "details": {"causes": [cause]}} + self.assertEqual(authorization_category(422, response), "immutable-uid") + for changed in ({"field": "subjects"}, {"reason": "Unexpected"}, {"message": "different error"}): + bad = {**response, "details": {"causes": [{**cause, **changed}]}} + self.assertEqual(authorization_category(422, bad), "unexpected-response") + self.assertEqual(authorization_category(403, response), "unexpected-response") + def test_controller_token_is_memory_only_no_admin_certificate_and_real_uid_is_required(self): account = {"metadata": {"uid": "actual-controller-uid"}} captured = [] From b8d130d8ca0a0d510ceb8304d244c5c2dc74dc17 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 00:01:58 +0200 Subject: [PATCH 23/62] fix(sre): preflight reviewed binding retirement with narrow authority Actual Kubernetes proof requires the named default reader bind permission even for subtractive updates. Add only that name; never grant wildcard bind, escalation, cluster-admin or custom-role authority. Prepare identical UID/RV-fenced patches and dry-run every retirement before any real mutation, preserving unrelated subjects and the legacy consumer on preflight failure. Keep API races fail-closed without claiming multi-resource rollback. Full hosted Rust and migration qualification remain required before readiness. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/testing/sre-authority.test.ts | 7 + controller/src/sre_authority.rs | 2 + controller/src/sre_authority/bindings.rs | 90 +++-- controller/src/sre_authority/privacy_tests.rs | 2 +- .../src/sre_authority/retirement_tests.rs | 365 ++++++++++++++++++ controller/src/sre_authority/tests.rs | 66 +++- .../kars/templates/sre-authority-rbac.yaml | 2 +- docs/how-to/sre-authority.md | 42 +- .../2026-09-08-sre-authority-prerequisite.md | 73 +++- tests/e2e/sre_authority/binding_probe.py | 10 +- tests/e2e/sre_authority/binding_probe_test.py | 3 + 11 files changed, 601 insertions(+), 61 deletions(-) create mode 100644 controller/src/sre_authority/retirement_tests.rs diff --git a/cli/src/testing/sre-authority.test.ts b/cli/src/testing/sre-authority.test.ts index 1dd59be25..bd439623d 100644 --- a/cli/src/testing/sre-authority.test.ts +++ b/cli/src/testing/sre-authority.test.ts @@ -38,6 +38,13 @@ describe("SRE authority chart and mutation integration",()=>{ const controller=docs.find(doc=>doc.kind==="ClusterRole"&&doc.metadata.name==="kars-sre-authority-controller"); expect(controller.rules.filter((rule:any)=>rule.resources.includes("karssreregistrations")) .every((rule:any)=>rule.verbs.every((verb:string)=>["get","list","watch","use"].includes(verb)))).toBe(true); + const privileged=controller.rules.filter((rule:any)=> + rule.verbs.some((verb:string)=>["bind","escalate","*"].includes(verb))); + expect(privileged).toEqual([{ + apiGroups:["rbac.authorization.k8s.io"],resources:["clusterroles"], + resourceNames:["kars-sre-private-diagnostics","kars-sre-action-author","kars-sre-router-renew","kars-sre-reader"], + verbs:["bind"], + }]); } }); diff --git a/controller/src/sre_authority.rs b/controller/src/sre_authority.rs index 738b826f4..fa42b5c80 100644 --- a/controller/src/sre_authority.rs +++ b/controller/src/sre_authority.rs @@ -13,6 +13,8 @@ pub(crate) mod pod; #[cfg(test)] mod privacy_tests; #[cfg(test)] +mod retirement_tests; +#[cfg(test)] mod tests; use crate::sre_registration::{KarsSRERegistration, NAME, RegistrationStatus}; diff --git a/controller/src/sre_authority/bindings.rs b/controller/src/sre_authority/bindings.rs index 086e458ed..36673f591 100644 --- a/controller/src/sre_authority/bindings.rs +++ b/controller/src/sre_authority/bindings.rs @@ -214,42 +214,64 @@ pub(super) async fn retire_legacy( reg: &KarsSRERegistration, bindings: &[Binding], ) -> Result<(), String> { - for binding in bindings { - if !binding.subjects.iter().any(legacy_subject) { - continue; - } - let subjects: Vec<_> = binding - .subjects - .iter() - .filter(|subject| !legacy_subject(subject)) - .cloned() - .collect(); - let patch = json!({ - "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version, - "annotations":{RETIRED:reg.metadata.uid}}, - "subjects":subjects, - }); - if binding.review.kind == "ClusterRoleBinding" { - Api::::all(client.clone()) - .patch( - &binding.review.name, - &PatchParams::default(), - &Patch::Merge(patch), + let plans: Vec<_> = bindings + .iter() + .filter(|binding| binding.subjects.iter().any(legacy_subject)) + .map(|binding| { + let subjects: Vec<_> = binding + .subjects + .iter() + .filter(|subject| !legacy_subject(subject)) + .cloned() + .collect(); + let patch = json!({ + "metadata":{"uid":binding.metadata.uid,"resourceVersion":binding.metadata.resource_version, + "annotations":{RETIRED:reg.metadata.uid}}, + "subjects":subjects, + }); + (binding, patch) + }) + .collect(); + // RBAC escalation checks also apply to subtractive binding updates. + // Preflight every exact patch before any write; this is not a transaction. + for dry_run in [true, false] { + let params = PatchParams { + dry_run, + ..Default::default() + }; + for (binding, patch) in &plans { + if binding.review.kind == "ClusterRoleBinding" { + Api::::all(client.clone()) + .patch(&binding.review.name, ¶ms, &Patch::Merge(patch)) + .await + .map_err(|e| { + api_error( + if dry_run { + "Preflight reviewed SRE ClusterRoleBinding retirement" + } else { + "Retire reviewed SRE ClusterRoleBinding" + }, + e, + ) + })?; + } else { + Api::::namespaced( + client.clone(), + binding.review.namespace.as_deref().unwrap(), ) + .patch(&binding.review.name, ¶ms, &Patch::Merge(patch)) .await - .map_err(|e| api_error("Retire reviewed SRE ClusterRoleBinding", e))?; - } else { - Api::::namespaced( - client.clone(), - binding.review.namespace.as_deref().unwrap(), - ) - .patch( - &binding.review.name, - &PatchParams::default(), - &Patch::Merge(patch), - ) - .await - .map_err(|e| api_error("Retire reviewed SRE RoleBinding", e))?; + .map_err(|e| { + api_error( + if dry_run { + "Preflight reviewed SRE RoleBinding retirement" + } else { + "Retire reviewed SRE RoleBinding" + }, + e, + ) + })?; + } } } Ok(()) diff --git a/controller/src/sre_authority/privacy_tests.rs b/controller/src/sre_authority/privacy_tests.rs index f52d2b003..c6b727112 100644 --- a/controller/src/sre_authority/privacy_tests.rs +++ b/controller/src/sre_authority/privacy_tests.rs @@ -25,7 +25,7 @@ fn ready() -> KarsSRERegistration { reg } -fn admission_ready(state: &Arc>) { +pub(super) fn admission_ready(state: &Arc>) { let mut state = state.lock().unwrap(); for name in admission::POLICIES { state.objects.insert(format!("/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/{name}"),json!({ diff --git a/controller/src/sre_authority/retirement_tests.rs b/controller/src/sre_authority/retirement_tests.rs new file mode 100644 index 000000000..e1b4a2b51 --- /dev/null +++ b/controller/src/sre_authority/retirement_tests.rs @@ -0,0 +1,365 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::privacy_tests::admission_ready; +use super::tests::{State, fixture, registration}; +use super::{bindings, reconcile}; +use crate::sre_registration::{ + BindingReview, ConsumerReview, KarsSRERegistration, RUNTIME_NAMESPACE, +}; +use k8s_openapi::api::rbac::v1::{ClusterRoleBinding, RoleBinding}; +use kube::{ + Api, + api::{Patch, PatchParams}, +}; +use serde_json::{Value, json}; +use std::sync::{Arc, Mutex}; + +const REG: &str = "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical"; +const CRBS: &str = "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings"; +const CONSUMER: &str = "/apis/apps/v1/namespaces/kars-sre/deployments/sre"; +const RETIRED: &str = "kars.azure.com/sre-legacy-retired"; + +fn reviews(state: &Arc>, kind: &str) -> (KarsSRERegistration, String) { + admission_ready(state); + let mut locked = state.lock().unwrap(); + locked.binding["metadata"]["annotations"] = json!({"e2e-retained":"yes"}); + let local = kind == "RoleBinding"; + let path = if local { + "/apis/rbac.authorization.k8s.io/v1/namespaces/kars-sre/rolebindings/second".into() + } else { + format!("{CRBS}/second") + }; + let mut second = json!({ + "apiVersion":"rbac.authorization.k8s.io/v1","kind":kind, + "metadata":{"name":"second","uid":"second-binding","resourceVersion":"7", + "annotations":{"e2e-retained":"yes"}}, + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":if local {"Role"} else {"ClusterRole"}, + "name":"custom-retirement-role"},"subjects":locked.binding["subjects"], + }); + if local { + second["metadata"]["namespace"] = RUNTIME_NAMESPACE.into(); + } + let mut reg = registration(); + reg.spec.legacy_bindings.push(BindingReview { + kind: kind.into(), + namespace: local.then(|| RUNTIME_NAMESPACE.into()), + name: "second".into(), + uid: "second-binding".into(), + resource_version: "7".into(), + role_ref: serde_json::from_value(second["roleRef"].clone()).unwrap(), + subjects: serde_json::from_value(second["subjects"].clone()).unwrap(), + }); + reg.spec.legacy_consumer = Some(ConsumerReview { + namespace: RUNTIME_NAMESPACE.into(), + name: "sre".into(), + uid: "consumer".into(), + resource_version: "1".into(), + }); + locked.objects.insert(path.clone(), second); + locked.objects.insert( + CONSUMER.into(), + json!({"apiVersion":"apps/v1","kind":"Deployment", + "metadata":{"name":"sre","namespace":"kars-sre","uid":"consumer","resourceVersion":"1"}, + "spec":{"replicas":1,"selector":{"matchLabels":{"app":"sre"}}, + "template":{"metadata":{},"spec":{"containers":[],"serviceAccountName":"sandbox"}}}}), + ); + locked + .objects + .insert(REG.into(), serde_json::to_value(®).unwrap()); + for (name, rule) in [ + ( + "kars-sre-private-diagnostics", + json!({"apiGroups":[""],"resources":["pods"],"verbs":["get"]}), + ), + ( + "kars-sre-action-author", + json!({"apiGroups":["kars.azure.com"],"resources":["karssreactions"],"verbs":["create"]}), + ), + ( + "kars-sre-router-renew", + json!({"apiGroups":["kars.azure.com"],"resources":["karssreregistrations"], + "resourceNames":["canonical"],"verbs":["get","renew"]}), + ), + ( + "custom-retirement-role", + json!({"apiGroups":[""],"resources":["limitranges"],"verbs":["get"]}), + ), + ] { + let role_path = if local && name == "custom-retirement-role" { + format!("/apis/rbac.authorization.k8s.io/v1/namespaces/kars-sre/roles/{name}") + } else { + format!("/apis/rbac.authorization.k8s.io/v1/clusterroles/{name}") + }; + locked + .objects + .insert(role_path, json!({"metadata":{"name":name},"rules":[rule]})); + } + (reg, path) +} + +fn survivors() -> Value { + json!([{"kind":"User","name":"unrelated","apiGroup":"rbac.authorization.k8s.io"}]) +} + +#[tokio::test] +async fn retirement_http_fixture_dry_runs_never_mutate_either_binding_store() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + let (_server, client, state) = fixture().await; + let (_reg, path) = reviews(&state, kind); + let (before, second) = { + let locked = state.lock().unwrap(); + (locked.binding.clone(), locked.objects[&path].clone()) + }; + let params = PatchParams { + dry_run: true, + ..Default::default() + }; + let patch = json!({"metadata":{"uid":"binding","resourceVersion":"1", + "annotations":{RETIRED:"registration"}},"subjects":survivors()}); + let projected = Api::::all(client.clone()) + .patch("legacy", ¶ms, &Patch::Merge(&patch)) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(projected).unwrap()["subjects"], + survivors() + ); + let patch = json!({"metadata":{"uid":"second-binding","resourceVersion":"7", + "annotations":{RETIRED:"registration"}},"subjects":survivors()}); + if kind == "RoleBinding" { + Api::::namespaced(client, RUNTIME_NAMESPACE) + .patch("second", ¶ms, &Patch::Merge(&patch)) + .await + .unwrap(); + } else { + Api::::all(client) + .patch("second", ¶ms, &Patch::Merge(&patch)) + .await + .unwrap(); + } + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before); + assert_eq!(locked.objects[&path], second); + assert_eq!(locked.binding_patches.len(), 2); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + } +} + +#[tokio::test] +async fn retirement_all_preflights_precede_identical_cas_writes_and_preserve_survivors() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, kind); + let before = state.lock().unwrap().objects.clone(); + let reviewed = bindings::review(&client, ®).await.unwrap(); + bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap(); + { + let locked = state.lock().unwrap(); + let patches = &locked.binding_patches; + assert_eq!(patches.len(), 4); + assert_eq!( + patches + .iter() + .map(|(_, dry_run, _)| *dry_run) + .collect::>(), + [true, true, false, false] + ); + for (preflight, write) in patches[..2].iter().zip(&patches[2..]) { + assert_eq!(preflight.0, write.0); + assert_eq!(preflight.2, write.2); + assert_eq!(preflight.2["subjects"], survivors()); + assert_eq!( + preflight.2["metadata"]["annotations"][RETIRED], + "registration" + ); + } + assert_eq!(patches[0].0, format!("{CRBS}/legacy")); + assert_eq!(patches[1].0, path); + assert_eq!(patches[0].2["metadata"]["uid"], "binding"); + assert_eq!(patches[0].2["metadata"]["resourceVersion"], "1"); + assert_eq!(patches[1].2["metadata"]["uid"], "second-binding"); + assert_eq!(patches[1].2["metadata"]["resourceVersion"], "7"); + for binding in [&locked.binding, &locked.objects[&path]] { + assert_eq!(binding["subjects"], survivors()); + assert_eq!(binding["metadata"]["annotations"]["e2e-retained"], "yes"); + assert_eq!(binding["metadata"]["annotations"][RETIRED], "registration"); + } + assert_eq!(locked.binding["metadata"]["resourceVersion"], "2"); + assert_eq!(locked.objects[&path]["metadata"]["resourceVersion"], "8"); + assert_eq!( + locked.binding["roleRef"], + serde_json::to_value(®.spec.legacy_bindings[0].role_ref).unwrap() + ); + assert_eq!(locked.objects[&path]["roleRef"], before[&path]["roleRef"]); + assert_eq!(locked.objects[CONSUMER], before[CONSUMER]); + assert!( + locked + .calls + .iter() + .all(|(method, _, _)| method == "GET" || method == "PATCH") + ); + } + let reviewed = bindings::review(&client, ®).await.unwrap(); + bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap(); + assert_eq!(state.lock().unwrap().binding_patches.len(), 4); + } +} + +#[tokio::test] +async fn retirement_second_binding_forbidden_preserves_all_grants_roles_and_consumer() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, kind); + let (before, mut before_objects) = { + let mut locked = state.lock().unwrap(); + locked + .binding_patch_errors + .insert((path.clone(), true), 403); + (locked.binding.clone(), locked.objects.clone()) + }; + let error = reconcile(&client, ®).await.unwrap_err(); + assert_eq!( + error, + format!("Preflight reviewed SRE {kind} retirement: Kubernetes status 403") + ); + assert!(!error.contains("PRIVATE_SENTINEL")); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before); + assert_eq!(locked.objects[REG]["status"]["phase"], "Blocked"); + let mut after_objects = locked.objects.clone(); + before_objects.remove(REG); + after_objects.remove(REG); + assert_eq!(after_objects, before_objects); + assert_eq!(locked.binding_patches.len(), 2); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + assert_eq!(locked.binding_patches[0].0, format!("{CRBS}/legacy")); + assert_eq!(locked.binding_patches[1].0, path); + assert!( + locked + .calls + .iter() + .all(|(method, _, _)| method == "GET" || method == "PATCH") + ); + assert!( + !locked + .calls + .iter() + .any(|(method, path, _)| method == "PATCH" && path == CONSUMER) + ); + } +} + +#[tokio::test] +async fn retirement_preflight_rejects_stale_uid_or_resource_version_before_any_write() { + for kind in ["ClusterRoleBinding", "RoleBinding"] { + for (field, changed, code) in [("uid", "replacement", 422), ("resourceVersion", "8", 409)] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, kind); + let reviewed = bindings::review(&client, ®).await.unwrap(); + let before = { + let mut locked = state.lock().unwrap(); + locked.objects.get_mut(&path).unwrap()["metadata"][field] = changed.into(); + (locked.binding.clone(), locked.objects.clone()) + }; + let error = bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap_err(); + assert_eq!( + error, + format!("Preflight reviewed SRE {kind} retirement: Kubernetes status {code}") + ); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before.0); + assert_eq!(locked.objects, before.1); + assert_eq!(locked.binding_patches.len(), 2); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + } + } +} + +#[tokio::test] +async fn retirement_preflight_propagates_other_api_failures_without_writes() { + for code in [409, 422, 500] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, "ClusterRoleBinding"); + let reviewed = bindings::review(&client, ®).await.unwrap(); + let before = { + let mut locked = state.lock().unwrap(); + locked.binding_patch_errors.insert((path, true), code); + (locked.binding.clone(), locked.objects.clone()) + }; + let error = bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap_err(); + assert_eq!( + error, + format!( + "Preflight reviewed SRE ClusterRoleBinding retirement: Kubernetes status {code}" + ) + ); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding, before.0); + assert_eq!(locked.objects, before.1); + assert!( + locked + .binding_patches + .iter() + .all(|(_, dry_run, _)| *dry_run) + ); + } +} + +#[tokio::test] +async fn retirement_write_phase_races_fail_closed_without_claiming_atomic_rollback() { + for code in [403, 409] { + let (_server, client, state) = fixture().await; + let (reg, path) = reviews(&state, "RoleBinding"); + let reviewed = bindings::review(&client, ®).await.unwrap(); + let before = { + let mut locked = state.lock().unwrap(); + locked + .binding_patch_errors + .insert((path.clone(), false), code); + locked.objects.clone() + }; + let error = bindings::retire_legacy(&client, ®, &reviewed) + .await + .unwrap_err(); + assert_eq!( + error, + format!("Retire reviewed SRE RoleBinding: Kubernetes status {code}") + ); + let locked = state.lock().unwrap(); + assert_eq!(locked.binding["subjects"], survivors()); + assert_eq!(locked.objects, before); + assert_eq!(locked.binding_patches.len(), 4); + assert_eq!( + locked + .binding_patches + .iter() + .map(|(_, dry_run, _)| *dry_run) + .collect::>(), + [true, true, false, false] + ); + } +} diff --git a/controller/src/sre_authority/tests.rs b/controller/src/sre_authority/tests.rs index b6d2a7519..c4e015b2e 100644 --- a/controller/src/sre_authority/tests.rs +++ b/controller/src/sre_authority/tests.rs @@ -47,7 +47,32 @@ fn binding() -> Value { fn api_error(code: u16) -> ResponseTemplate { ResponseTemplate::new(code).set_body_json(json!({"apiVersion":"v1","kind":"Status","status":"Failure", - "code":code,"reason":if code==404 {"NotFound"} else {"Forbidden"},"message":"PRIVATE_SENTINEL"})) + "code":code,"reason":match code {404=>"NotFound",409=>"Conflict",422=>"Invalid",_=>"Forbidden"}, + "message":"PRIVATE_SENTINEL"})) +} + +fn patched_binding(existing: &Value, patch: &Value, dry_run: bool) -> Value { + let mut value = existing.clone(); + value["subjects"] = patch["subjects"].clone(); + if let Some(annotations) = patch["metadata"]["annotations"].as_object() { + if !value["metadata"]["annotations"].is_object() { + value["metadata"]["annotations"] = json!({}); + } + for (key, annotation) in annotations { + value["metadata"]["annotations"][key] = annotation.clone(); + } + } + if !dry_run { + value["metadata"]["resourceVersion"] = (existing["metadata"]["resourceVersion"] + .as_str() + .unwrap() + .parse::() + .unwrap() + + 1) + .to_string() + .into(); + } + value } pub(super) struct State { @@ -60,6 +85,8 @@ pub(super) struct State { pub(super) objects: BTreeMap, pub(super) watch_allowed: Option<(Option, Option)>, pub(super) metadata_requests: Vec, + pub(super) binding_patches: Vec<(String, bool, Value)>, + pub(super) binding_patch_errors: BTreeMap<(String, bool), u16>, } pub(super) async fn fixture() -> (MockServer, Client, Arc>) { @@ -73,6 +100,8 @@ pub(super) async fn fixture() -> (MockServer, Client, Arc>) { objects: BTreeMap::new(), watch_allowed: None, metadata_requests: Vec::new(), + binding_patches: Vec::new(), + binding_patch_errors: BTreeMap::new(), })); let server = MockServer::start().await; let handler = state.clone(); @@ -80,7 +109,14 @@ pub(super) async fn fixture() -> (MockServer, Client, Arc>) { let mut state=handler.lock().unwrap(); let path=request.url.path(); let body:Value=request.body_json().unwrap_or(Value::Null); + let dry_run=request.url.query_pairs().any(|(key,value)|key=="dryRun" && value=="All"); state.calls.push((request.method.to_string(),path.into(),body.clone())); + if request.method == "PATCH" && body["subjects"].is_array() { + state.binding_patches.push((path.into(),dry_run,body.clone())); + if let Some(code)=state.binding_patch_errors.get(&(path.into(),dry_run)) { + return api_error(*code); + } + } if request.method == "GET" && path=="/api/v1/namespaces/kars-sre/secrets" { state.metadata_requests.push(request.headers.get("accept").unwrap().to_str().unwrap().into()); } @@ -116,11 +152,12 @@ pub(super) async fn fixture() -> (MockServer, Client, Arc>) { return ResponseTemplate::new(200).set_body_json(value.clone()); } if request.method == "PATCH" && body["subjects"].is_array() && state.objects.contains_key(path) { - let value=state.objects.get_mut(path).unwrap(); - if body["metadata"]["uid"]!=value["metadata"]["uid"] || - body["metadata"]["resourceVersion"]!=value["metadata"]["resourceVersion"] {return api_error(409);} - value["subjects"]=body["subjects"].clone(); - return ResponseTemplate::new(200).set_body_json(value.clone()); + let value=&state.objects[path]; + if body["metadata"]["uid"]!=value["metadata"]["uid"] {return api_error(422);} + if body["metadata"]["resourceVersion"]!=value["metadata"]["resourceVersion"] {return api_error(409);} + let value=patched_binding(value,&body,dry_run); + if !dry_run {state.objects.insert(path.into(),value.clone());} + return ResponseTemplate::new(200).set_body_json(value); } let value=match (request.method.as_str(),path) { ("GET","/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical")=>serde_json::to_value(registration()).unwrap(), @@ -136,15 +173,18 @@ pub(super) async fn fixture() -> (MockServer, Client, Arc>) { .map(|(_,object)|json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":object["metadata"]})) .collect::>()}), ("GET","/apis/rbac.authorization.k8s.io/v1/clusterrolebindings")=>json!({ - "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBindingList","metadata":{},"items":[state.binding]}), + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRoleBindingList","metadata":{}, + "items":std::iter::once(state.binding.clone()).chain(state.objects.values() + .filter(|object|object["kind"]=="ClusterRoleBinding").cloned()).collect::>()}), ("GET","/apis/rbac.authorization.k8s.io/v1/rolebindings")=>json!({ - "apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBindingList","metadata":{},"items":[]}), + "apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBindingList","metadata":{}, + "items":state.objects.values().filter(|object|object["kind"]=="RoleBinding").cloned().collect::>()}), ("PATCH","/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/legacy")=>{ - if body["metadata"]["uid"]!=state.binding["metadata"]["uid"] - || body["metadata"]["resourceVersion"]!=state.binding["metadata"]["resourceVersion"] {return api_error(409)} - state.binding["subjects"]=body["subjects"].clone(); - state.binding["metadata"]["annotations"]=body["metadata"]["annotations"].clone(); - state.binding.clone() + if body["metadata"]["uid"]!=state.binding["metadata"]["uid"] {return api_error(422)} + if body["metadata"]["resourceVersion"]!=state.binding["metadata"]["resourceVersion"] {return api_error(409)} + let value=patched_binding(&state.binding,&body,dry_run); + if !dry_run {state.binding=value.clone();} + value } ("POST","/apis/authorization.k8s.io/v1/subjectaccessreviews")=>json!({ "apiVersion":"authorization.k8s.io/v1","kind":"SubjectAccessReview", diff --git a/deploy/helm/kars/templates/sre-authority-rbac.yaml b/deploy/helm/kars/templates/sre-authority-rbac.yaml index 27ad29555..819aef1c2 100644 --- a/deploy/helm/kars/templates/sre-authority-rbac.yaml +++ b/deploy/helm/kars/templates/sre-authority-rbac.yaml @@ -112,7 +112,7 @@ rules: verbs: ["create", "patch", "update", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["clusterroles"] - resourceNames: ["kars-sre-private-diagnostics", "kars-sre-action-author", "kars-sre-router-renew"] + resourceNames: ["kars-sre-private-diagnostics", "kars-sre-action-author", "kars-sre-router-renew", "kars-sre-reader"] verbs: ["bind"] - apiGroups: ["authorization.k8s.io"] resources: ["subjectaccessreviews"] diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index e89309d09..d26426f89 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -72,11 +72,43 @@ and wait for migration: kars sre authority migrate --namespace kars-system --release kars ``` -The controller validates the full review set before mutation, removes only the -legacy SRE subject from the reviewed bindings using UID/resourceVersion -preconditions, preserves unrelated subjects, and stops the reviewed old -consumer. Broad group grants or unreviewed/custom resources stop migration; -the operator must restructure those grants explicitly. +The controller validates the full review set, then submits every planned +retirement PATCH to the API server with `dryRun=All` before applying any of +them. Each dry-run and real PATCH uses the identical reviewed UID/resourceVersion, +retirement annotation and surviving subjects. Only the legacy SRE subject is +removed; unrelated subjects and the binding's roleRef remain intact. The +reviewed old consumer is stopped only after all binding retirements succeed. +Broad group grants and unreviewed resources stop migration; the operator must +restructure those grants explicitly. + +### Binding authorization and custom roles + +Kubernetes applies RBAC privilege-escalation checks even to a **subtractive** +ClusterRoleBinding or RoleBinding update. Ordinary `patch` permission and +registrar `use` are not sufficient: the controller must already hold the +referenced permissions or have the appropriate named `bind` permission. +The chart grants `bind` on exactly `kars-sre-reader`, +`kars-sre-private-diagnostics`, `kars-sre-action-author` and `kars-sre-router-renew`. +It does not grant wildcard `bind`, `escalate`, cluster-admin, or automatic +authority over custom roles. + +A reviewed custom role can therefore fail server-side preflight. A failure +such as `Preflight reviewed SRE ClusterRoleBinding retirement: Kubernetes status 403` +leaves all legacy bindings and the consumer unchanged on that initial attempt. +The controller relies on the actual API server's RBAC, admission and validation +decisions; it does not simulate those checks or grant itself missing authority. +An operator must inspect the referenced role and explicitly resolve the grant: +restructure or retire it, or separately authorize only the exact required +role-specific binding permission after review. Never add broad bind/escalate +rights merely to make migration pass. Refresh enrollment UID/resourceVersion +reviews if an operator changes the reviewed resources. + +Preflight is **not a multi-resource transaction**. RBAC, admission or object +versions can change before a real PATCH. Every real write still enforces its +original UID/resourceVersion and current API authorization, and errors stop the +attempt. Earlier successful retirements can remain applied; they are recognized +on retry, not rolled back by restoring old privileges. The consumer is not +stopped and private credentials are not issued while retirement remains incomplete. Real Kubernetes authorization reviews must deny the old SRE principal Secret get/list/watch access before private credentials are issued. The shared diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 2e907d802..67beca344 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -1,7 +1,8 @@ # Security audit — registered SRE credential authority -Status: candidate under qualification; the latest schema repair still requires -Rust and full controller/migration execution. **Not a sign-off.** +Status: candidate under qualification; the local named-reader-bind and +retirement-preflight repair requires Rust and full controller/migration +execution. **Not a sign-off.** ## Scope and trust root @@ -37,12 +38,74 @@ the actual full-harness discovery command. The new Rust schema/wire regressions, existing Helm/Rust drift test and full fatal SRE migration remain required before readiness. A CI run, not the unbuilt local repair, supplies that next evidence. +## Subtractive binding authorization: real API evidence and local repair + +The image-free hosted Kind experiment at +https://github.com/Azure/kars/actions/runs/34280933428/job/102245425114 +ran against immutable head `c1d14fad4bbaeeda6832d278943ba9bba4e9130e`. +Artifact `sre-crd-schema-34280933428` contains +`bootstrap-binding-retirement.json`; no bearer tokens or response bodies are +included. SelfSubjectReview verified the short-lived controller ServiceAccount +bearer's actual UID, with no admin client certificate or impersonation fallback. + +The controller already had ordinary ClusterRoleBinding PATCH and canonical +registrar `use`, but lacked named `bind` on the historical `kars-sre-reader`. +The exact reviewed UID/resourceVersion-fenced subtractive PATCH returned an +explicit Kubernetes RBAC permissions-not-held **403**. Adding only a disposable +`bind` grant for that one ClusterRole changed the identical dry-run to **200**; +removing the test grant restored **403**. A separately reviewed custom role +remained **403** throughout. Wrong UID produced the specific immutable-UID +**422** validation error, and stale resourceVersion produced **409 Conflict**. +Every persistent binding, unrelated subject, role and consumer remained +unchanged; the temporary permission was removed. All fourteen SRE policies +remained observed, warning-free and Deny-bound, and all nine existing +ordinary/private/Pending-only admission cases passed. No workload image was +executed and no full migration readiness was claimed. + +The local production candidate adds only `kars-sre-reader` to the controller's +existing three-name ClusterRole `bind` allowlist, yielding four exact names. +It introduces no wildcard, `escalate`, cluster-admin, default agent grant, or automatic custom +role permission. Retirement prepares immutable patches, dry-runs **all** of them +through the API server, then applies those same patches only if every preflight +succeeds. An initial permission, admission or validation failure is reported +with safe `Preflight` context before any binding retirement or consumer stop. +Custom-role failures require explicit operator remediation and fresh reviews +where needed; the controller never self-grants the missing permission. + +This is not a multi-resource transaction. A later real write can fail if +permissions or versions change after preflight. Prior completed retirements can +remain, errors stop further progress, and retries recognize retired bindings +without restoring old grants. Rust HTTP fixtures now distinguish `dryRun=All` +from real PATCHes and do not mutate persistent fixture state on dry-run. +New regressions cover both binding kinds, a forbidden second preflight, +preserved consumers/custom roles, UID/RV fences, ordering, idempotence and +post-preflight failures. The updated fast API probe requires the shipped reader +bind to work before and after its now-redundant test grant; it does not describe +that new baseline as another 403/200/403 experiment. + +The new Rust changes have not been compiled or executed by this task because +the shared Cargo lease belongs to another qualification run. Source review and +parent approval remain required before public production push; Rust +qualification and the complete fatal migration remain required before claiming +readiness. This section is evidence and candidate documentation, not audit +sign-off or permission to deploy. + +Local checks for this candidate passed all 53 Python harness tests, all five +targeted CLI/Helm authority tests (including the exact four-name bind list and +unchanged default private-grant assertions), standalone scoped rustfmt, shell +syntax and whitespace checks. The six new Rust regression functions are in +`sre_authority::retirement_tests`; parent qualification should also run the +existing `sre_authority::tests` and `sre_authority::privacy_tests` because they +share the corrected HTTP fixture. No Cargo invocation, dependency change, +public production push or new audit signature was performed. + ## Boundaries implemented - Exact source, controller/release, and runtime namespace UIDs are checked live. -- Reviewed legacy grants retire with UID/resourceVersion preconditions. - Unrelated subjects/resources are preserved; custom/group ambiguity blocks - before migration mutations. +- Reviewed legacy grants retire with UID/resourceVersion preconditions after + all exact server-side retirement dry-runs pass. Unrelated subjects/resources + are preserved; group/unreviewed ambiguity or custom-role preflight denial + blocks before initial migration mutations. - Shared live authorization reviews require Secret get/list/watch denial in namespace and cluster scope, including protected name-restricted grants, before issuance and proxy forwarding. Old status booleans are insufficient. diff --git a/tests/e2e/sre_authority/binding_probe.py b/tests/e2e/sre_authority/binding_probe.py index 3da49f575..203f60df3 100644 --- a/tests/e2e/sre_authority/binding_probe.py +++ b/tests/e2e/sre_authority/binding_probe.py @@ -250,6 +250,12 @@ def prove(root, port, state, objects, report): "bindCustom": api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", CUSTOM), "getCustomLimitRanges": api.allowed("", "limitranges", "get"), "getReaderNodeMetrics": api.allowed("metrics.k8s.io", "nodes", "get")} + require(baseline_bind, "Shipped controller lacks the exact named legacy reader bind permission") + facts["shippedReaderBindVerified"] = True + # The historical 403/200/403 experiment is retained in its immutable CI + # artifact. With the repaired chart, this extra grant is deliberately + # redundant: removing it must leave the shipped reader bind authorized. + facts["temporaryReaderBindGrant"] = "redundant-with-shipped-permission" require(not facts["authorization"]["bindCustom"] and not facts["authorization"]["getCustomLimitRanges"], "Custom fixture is not outside the controller's current authority") @@ -292,7 +298,7 @@ def probe(binding, phase, expected, stale=None): unchanged() reader, custom = state["bindings"] - probe(reader, "baseline", 200 if baseline_bind else 403) + probe(reader, "shipped-baseline", 200) probe(custom, "baseline-custom", 403) extra = [] try: @@ -322,7 +328,7 @@ def probe(binding, phase, expected, stale=None): "preconditions": {k: obj["metadata"][k] for k in ("uid", "resourceVersion")}}) require(code in (200, 202), "Exact named bind fixture cleanup failed") observe_bind(api, baseline_bind) - probe(reader, "named-bind-removed", 200 if baseline_bind else 403) + probe(reader, "temporary-bind-removed-shipped-bind-remains", 200) facts["unchangedPoliciesRolesBindingsConsumer"] = True facts["testBindGrantRemoved"] = True report(facts) diff --git a/tests/e2e/sre_authority/binding_probe_test.py b/tests/e2e/sre_authority/binding_probe_test.py index ff60cebfc..ff0527b7b 100644 --- a/tests/e2e/sre_authority/binding_probe_test.py +++ b/tests/e2e/sre_authority/binding_probe_test.py @@ -123,6 +123,9 @@ def test_fast_hosted_gate_keeps_prior_admission_cases_and_adds_real_bind_experim self.assertIn('"resourceNames": [READER], "verbs": ["bind"]', probe) self.assertNotIn('"verbs": ["escalate"]', probe) self.assertNotIn('"resourceNames": ["*"]', probe) + self.assertIn('require(baseline_bind, "Shipped controller lacks', probe) + self.assertIn('"temporary-bind-removed-shipped-bind-remains", 200', probe) + self.assertIn('"redundant-with-shipped-permission"', probe) if __name__ == "__main__": From 105da37a540adb274c47f8a49ad0bc084a88c971 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 12:16:58 +0200 Subject: [PATCH 24/62] test(e2e): establish immutable legacy CRDs before Helm hooks Create only the 18 CRDs rendered from the exact historical archive after validating inventory, schema content and absence. Attach legitimate release ownership and wait for Established plus API discovery before the unchanged historical post-install hook. Reject adoption, conflicting creates and unrelated API errors; preserve migration ordering and production guards. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/fixtures.py | 15 +- tests/e2e/sre_authority/legacy_crds.py | 151 ++++++++++++ tests/e2e/sre_authority/legacy_crds_test.py | 244 ++++++++++++++++++++ 3 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/sre_authority/legacy_crds.py create mode 100644 tests/e2e/sre_authority/legacy_crds_test.py diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py index 6653a2939..b9d8e2392 100644 --- a/tests/e2e/sre_authority/fixtures.py +++ b/tests/e2e/sre_authority/fixtures.py @@ -11,6 +11,7 @@ enrollment_json, fingerprint, require, ) from .credential_paths import seed_privacy_gaps +from .legacy_crds import bootstrap_legacy_crds from .registration_schema import create_registration_crd LEGACY_COMMIT = "8b206065608593667a40665b3f48225ef9ce278d" @@ -58,7 +59,8 @@ def prepare_legacy(h): data = subprocess.run(["git", "archive", LEGACY_COMMIT, "deploy/helm/kars"], cwd=h.root, capture_output=True, timeout=30, check=True).stdout destination = h.work / "legacy-chart" - destination.mkdir(exist_ok=True) + require(not destination.exists(), "Historical fixture extraction destination already exists") + destination.mkdir() with tarfile.open(fileobj=io.BytesIO(data)) as chart: for member in chart.getmembers(): require((member.name.startswith("deploy/helm/kars/") @@ -66,6 +68,14 @@ def prepare_legacy(h): and ".." not in member.name.split("/") and (member.isdir() or member.isfile()), "Unsafe historical chart archive") chart.extractall(destination, filter="data") + sources = { + member.name.rsplit("/", 1)[-1]: chart.extractfile(member).read().decode() + for member in chart.getmembers() + if member.isfile() and member.name.startswith("deploy/helm/kars/templates/crd") + } + # The historical chart puts CRDs in templates/, so Helm's CRD-directory + # readiness does not order its post-install ToolPolicy hook. + bootstrap_legacy_crds(h, destination / "deploy/helm/kars", sources) h.run(["helm", "install", "kars", str(destination / "deploy/helm/kars"), "--kube-context", CONTEXT, "--namespace", SYSTEM, "--create-namespace", "--set", "controller.replicas=0", "--set", "controller.image.repository=kars-controller", @@ -75,6 +85,9 @@ def prepare_legacy(h): "--set", "sandbox.image.tag=dev", "--set-string", f"runtimes.hermes.image={STANDIN}", "--set", "sre.enabled=false", "--set-string", "inferenceRouter.azure.openai.endpoint=https://e2e-fake.invalid/", "--set-string", "foundry.endpoint=https://e2e-fake.invalid/"], timeout=180) + require(h.get("toolpolicy", "kars-default", SYSTEM) is not None, + "Historical Helm post-install ToolPolicy hook did not create its policy") + h.passed("Historical Helm initial install and unchanged ToolPolicy post-install hook completed") h.k("wait", "--for=condition=Established", "crd/karssandboxes.kars.azure.com", "--timeout=60s", timeout=70) h.run(["helm", "upgrade", "kars", str(destination / "deploy/helm/kars"), "--kube-context", CONTEXT, "--namespace", SYSTEM, "--reuse-values", "--set", "sre.enabled=true"], timeout=120) diff --git a/tests/e2e/sre_authority/legacy_crds.py b/tests/e2e/sre_authority/legacy_crds.py new file mode 100644 index 000000000..0fbd6f0cd --- /dev/null +++ b/tests/e2e/sre_authority/legacy_crds.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Create-only readiness for CRDs from the immutable historical Helm fixture.""" + +import copy +import json + +from .common import SYSTEM, require +from .registration_schema import CRD_NAME, CRD_PATH + +# This is the complete historical inventory, not the current chart's CRDs. +CRDS = { + "crd-a2aagent.yaml": ("a2aagents", "A2AAgent", "Namespaced"), + "crd-egressapproval.yaml": ("egressapprovals", "EgressApproval", "Namespaced"), + "crd-inferencepolicy.yaml": ("inferencepolicies", "InferencePolicy", "Namespaced"), + "crd-karsapproval.yaml": ("karsapprovals", "KarsApproval", "Namespaced"), + "crd-karsauthconfig.yaml": ("karsauthconfigs", "KarsAuthConfig", "Cluster"), + "crd-karseval.yaml": ("karsevals", "KarsEval", "Namespaced"), + "crd-karsmemory.yaml": ("karsmemories", "KarsMemory", "Namespaced"), + "crd-karsprofile.yaml": ("karsprofiles", "KarsProfile", "Namespaced"), + "crd-karsreceipt.yaml": ("karsreceipts", "KarsReceipt", "Namespaced"), + "crd-karsskill.yaml": ("karsskills", "KarsSkill", "Namespaced"), + "crd-karssreaction.yaml": ("karssreactions", "KarsSREAction", "Namespaced"), + "crd-karstask.yaml": ("karstasks", "KarsTask", "Namespaced"), + "crd-karsteam.yaml": ("karsteams", "KarsTeam", "Namespaced"), + "crd-mcpserver.yaml": ("mcpservers", "McpServer", "Namespaced"), + "crd-toolpolicy.yaml": ("toolpolicies", "ToolPolicy", "Namespaced"), + "crd-trustgraph.yaml": ("trustgraphs", "TrustGraph", "Cluster"), + "crd.yaml": ("karssandboxes", "KarsSandbox", "Namespaced"), +} +# The historical crd.yaml contains both KarsSandbox and KarsPairing. +IDENTITIES = (*CRDS.values(), ("karspairings", "KarsPairing", "Namespaced")) +GROUP_VERSION = "kars.azure.com/v1alpha1" + + +def validate_rendered_crds(rendered, historical): + require(isinstance(historical, dict) and set(historical) == set(CRDS), + "Historical CRD inventory differs from the immutable baseline") + require(isinstance(rendered, list) and len(rendered) == len(IDENTITIES), + "Historical render omitted or added resources") + expected = {} + for filename, identity in CRDS.items(): + identities = [identity, IDENTITIES[-1]] if filename == "crd.yaml" else [identity] + require(isinstance(historical[filename], list) and len(historical[filename]) == len(identities), + "Historical template has an unexpected number of CRDs") + for obj, expected_identity in zip(historical[filename], identities): + validate_historical_crd(obj, expected_identity) + expected[obj["metadata"]["name"]] = obj + seen = set() + for obj in rendered: + require(isinstance(obj, dict), "Historical render contained a non-object") + name = obj.get("metadata", {}).get("name") + require(name in expected and name not in seen and obj == expected[name], + "Rendered CRD content differs from its immutable historical template") + seen.add(name) + return rendered + + +def validate_historical_crd(obj, identity): + plural, kind, scope = identity + name = f"{plural}.kars.azure.com" + require(isinstance(obj, dict) and set(obj) == {"apiVersion", "kind", "metadata", "spec"} + and obj["apiVersion"] == "apiextensions.k8s.io/v1" + and obj["kind"] == "CustomResourceDefinition" + and obj["metadata"].get("name") == name + and not set(obj["metadata"]) - {"name", "labels"} + and obj["spec"].get("group") == "kars.azure.com" + and obj["spec"].get("scope") == scope + and obj["spec"].get("names", {}).get("plural") == plural + and obj["spec"].get("names", {}).get("kind") == kind, + "Historical CRD identity or ownership differs from the immutable baseline") + versions = obj["spec"].get("versions", []) + require(len(versions) == 1 and versions[0].get("name") == "v1alpha1" + and versions[0].get("served") is True and versions[0].get("storage") is True + and versions[0].get("schema", {}).get("openAPIV3Schema", {}).get("type") == "object", + "Historical CRD version/schema differs from the immutable baseline") + + +def render_legacy_crds(h, chart, sources): + require(set(sources) == set(CRDS) + and {path.name for path in (chart / "templates").glob("crd*.yaml")} == set(CRDS), + "Historical archive contains an unexpected CRD inventory") + for filename, content in sources.items(): + require("{{" not in content and (chart / "templates" / filename).read_text() == content, + "Historical CRD template changed after immutable archive extraction") + args = ["helm", "template", "kars", str(chart), "--namespace", SYSTEM, + "--set", "controller.replicas=0", "--set", "sre.enabled=false"] + for filename in CRDS: + args += ["--show-only", f"templates/{filename}"] + rendered = h.run(args, timeout=45) + converter = """ +const y=require('node:module').createRequire(process.cwd()+'/cli/package.json')('yaml'); +const input=JSON.parse(require('fs').readFileSync(0,'utf8')); +function documents(text) { + return y.parseAllDocuments(text).map(doc => { + if (doc.errors.length) throw Error('Invalid historical YAML'); + return doc.toJSON(); + }).filter(doc => doc !== null); +} +const historical=Object.fromEntries(Object.entries(input.sources).map(([name,text]) => { + return [name,documents(text)]; +})); +console.log(JSON.stringify({historical,rendered:documents(input.rendered)})); +""" + parsed = json.loads(h.run(["node", "-e", converter], + data=json.dumps({"sources": sources, "rendered": rendered}), timeout=20)) + return validate_rendered_crds(parsed["rendered"], parsed["historical"]) + + +def bootstrap_legacy_crds(h, chart, sources): + objects = render_legacy_crds(h, chart, sources) + # Preflight every absence before the first CREATE. An existing object, + # including one claiming this release, is never adopted or patched. + for name in [CRD_NAME] + [obj["metadata"]["name"] for obj in objects]: + response = h.api("GET", f"{CRD_PATH}/{name}", status=404) + body = response.json() + require(body.get("kind") == "Status" and body.get("reason") == "NotFound", + "Historical CRD absence was not a Kubernetes NotFound") + for original in objects: + obj = copy.deepcopy(original) + obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" + obj["metadata"]["annotations"] = { + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM, + } + h.create(obj) + h.k("wait", "--for=condition=Established", + *[f'crd/{obj["metadata"]["name"]}' for obj in objects], "--timeout=60s", timeout=70) + + def discovered(): + response = h.api("GET", f"/apis/{GROUP_VERSION}", status=(200, 404)) + body = response.json() + if response.status_code == 404: + require(body.get("kind") == "Status" and body.get("reason") == "NotFound", + "Historical API discovery returned an unexpected response") + return False + require(body.get("kind") == "APIResourceList" and body.get("groupVersion") == GROUP_VERSION + and isinstance(body.get("resources"), list), + "Historical API discovery returned an unexpected resource list") + resources = {item["name"]: item for item in body["resources"]} + for plural, kind, scope in IDENTITIES: + if plural not in resources: + return False + item = resources[plural] + require(item.get("kind") == kind and item.get("namespaced") == (scope == "Namespaced") + and {"create", "get", "list", "watch"}.issubset(item.get("verbs", [])), + "Historical API discovery differs from the immutable CRD identity") + return True + + h.poll("Historical CRD API discovery before Helm hooks", discovered, seconds=60) + h.passed("All 18 unchanged historical CRDs created with Helm ownership, Established and discovered before initial hooks") diff --git a/tests/e2e/sre_authority/legacy_crds_test.py b/tests/e2e/sre_authority/legacy_crds_test.py new file mode 100644 index 000000000..275917872 --- /dev/null +++ b/tests/e2e/sre_authority/legacy_crds_test.py @@ -0,0 +1,244 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure historical fixture checks, not a substitute for hosted Kind acceptance.""" + +import copy +from pathlib import Path +import types +import unittest +from unittest.mock import Mock, patch + +from sre_authority.legacy_crds import ( + CRDS, GROUP_VERSION, IDENTITIES, bootstrap_legacy_crds, render_legacy_crds, validate_rendered_crds, +) +from sre_authority.registration_schema import CRD_NAME + + +def historical_objects(): + def obj(plural, kind, scope): + return { + "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", + "metadata": {"name": f"{plural}.kars.azure.com", "labels": {"app.kubernetes.io/name": "kars"}}, + "spec": {"group": "kars.azure.com", "scope": scope, "names": {"plural": plural, "kind": kind}, + "versions": [{"name": "v1alpha1", "served": True, "storage": True, + "schema": {"openAPIV3Schema": {"type": "object"}}}]}, + } + result = {filename: [obj(*identity)] for filename, identity in CRDS.items()} + result["crd.yaml"].append(obj(*IDENTITIES[-1])) + return result + + +def flattened(historical): + return [obj for objects in historical.values() for obj in objects] + + +def discovery(): + return {"kind": "APIResourceList", "groupVersion": GROUP_VERSION, "resources": [ + {"name": plural, "kind": kind, "namespaced": scope == "Namespaced", + "verbs": ["create", "get", "list", "watch"]} + for plural, kind, scope in IDENTITIES + ]} + + +class FixtureHarness: + def __init__(self): + self.events = [] + self.responses = [(200, discovery())] + self.existing = None + self.create_failure = False + self.wait_failure = False + + def api(self, method, path, *, status): + self.events.append(("api", method, path)) + if path == f"/apis/{GROUP_VERSION}": + code, body = self.responses.pop(0) + elif path.endswith("/" + str(self.existing)): + code, body = 200, {"kind": "CustomResourceDefinition"} + else: + code, body = 404, {"kind": "Status", "reason": "NotFound"} + if code not in (status if isinstance(status, tuple) else (status,)): + raise AssertionError(f"HTTP {code}") + return types.SimpleNamespace(status_code=code, json=lambda: body) + + def create(self, obj): + self.events.append(("create", obj)) + if self.create_failure: + raise AssertionError("HTTP 409") + + def k(self, *args, **kwargs): + self.events.append(("wait", args, kwargs)) + if self.wait_failure: + raise AssertionError("wait-timeout") + + def poll(self, label, predicate, *, seconds): + self.events.append(("poll", label, seconds)) + for _ in range(3): + if predicate(): + return True + raise AssertionError("bounded discovery deadline") + + def passed(self, message): + self.events.append(("passed", message)) + + +class LegacyCRDTests(unittest.TestCase): + def bootstrap(self, h): + objects = flattened(historical_objects()) + with patch("sre_authority.legacy_crds.render_legacy_crds", return_value=objects): + bootstrap_legacy_crds(h, Path("unused-test-chart"), {}) + return objects + + def test_render_requires_exact_historical_content_not_just_a_matching_name(self): + historical = historical_objects() + rendered = flattened(copy.deepcopy(historical)) + self.assertEqual(validate_rendered_crds(rendered, historical), rendered) + rendered[0]["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["description"] = "changed" + with self.assertRaises(AssertionError): + validate_rendered_crds(rendered, historical) + + def test_current_registration_extra_missing_duplicate_and_foreign_resources_are_rejected(self): + historical = historical_objects() + valid = flattened(copy.deepcopy(historical)) + registration = copy.deepcopy(valid[0]) + registration["metadata"]["name"] = CRD_NAME + for rendered in (valid[:-1], valid + [registration], [registration] + valid[1:], + [valid[0]] + valid[:-1], [{"kind": "Secret"}] + valid[1:]): + with self.subTest(count=len(rendered)), self.assertRaises(AssertionError): + validate_rendered_crds(rendered, historical) + with self.assertRaises(AssertionError): + validate_rendered_crds(valid, {**historical, "crd-karssreregistration.yaml": registration}) + + def test_historical_identity_schema_and_preexisting_ownership_are_strict(self): + def changed(section, key, value): + obj = historical_objects() + obj["crd.yaml"][0][section][key] = value + return obj + invalid = [ + changed("metadata", "name", "other.kars.azure.com"), + changed("metadata", "annotations", {"meta.helm.sh/release-name": "other"}), + changed("metadata", "ownerReferences", [{"uid": "foreign"}]), + changed("spec", "group", "foreign.example"), + changed("spec", "scope", "Cluster"), + changed("spec", "names", {"plural": "karssandboxes", "kind": "Secret"}), + changed("spec", "versions", []), + ] + for historical in invalid: + with self.subTest(historical=historical["crd.yaml"]), self.assertRaises(AssertionError): + validate_rendered_crds(flattened(historical), historical) + + def test_changed_extracted_source_and_unexpected_inventory_fail_before_render(self): + h = Mock() + # A small path double avoids filesystem writes in these pure checks. + class Chart: + def __truediv__(self, _name): + return self + + def glob(self, _pattern): + return [types.SimpleNamespace(name=filename) for filename in CRDS] + + def read_text(self): + return "changed" + for sources in ({filename: "original" for filename in CRDS}, + {filename: "{{ templated }}" for filename in CRDS}, + {"crd-karssreregistration.yaml": "unexpected"}): + with self.assertRaises(AssertionError): + render_legacy_crds(h, Chart(), sources) + h.run.assert_not_called() + + def test_create_only_ownership_and_all_readiness_precede_hooks(self): + h = FixtureHarness() + originals = self.bootstrap(h) + names = [obj["metadata"]["name"] for obj in originals] + self.assertEqual(len(names), 18) + self.assertNotIn(CRD_NAME, names) + self.assertEqual([event[2].rsplit("/", 1)[-1] for event in h.events[:19]], + [CRD_NAME] + names) + creates = [event[1] for event in h.events if event[0] == "create"] + self.assertEqual(len(creates), 18) + for original, created in zip(originals, creates): + self.assertNotIn("annotations", original["metadata"]) + self.assertEqual(created["spec"], original["spec"]) + self.assertEqual(created["metadata"]["labels"]["app.kubernetes.io/managed-by"], "Helm") + self.assertEqual(created["metadata"]["annotations"], { + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": "kars-system"}) + wait = next(event for event in h.events if event[0] == "wait") + self.assertEqual(wait[1], ("wait", "--for=condition=Established", + *[f"crd/{name}" for name in names], "--timeout=60s")) + self.assertLess(h.events.index(wait), next(i for i, e in enumerate(h.events) if e[0] == "poll")) + self.assertEqual(h.events[-1][0], "passed") + fixture = (Path(__file__).with_name("fixtures.py")).read_text() + self.assertLess(fixture.index("bootstrap_legacy_crds(h,"), fixture.index('h.run(["helm", "install"')) + self.assertLess(fixture.index('h.run(["helm", "install"'), fixture.index('h.get("toolpolicy"')) + for bypass in ("--take-ownership", "--force", "--validate=false", "--no-hooks", "governance.enabled=false"): + self.assertNotIn(bypass, fixture) + + def test_existing_foreign_or_even_release_named_crd_is_never_adopted(self): + for name in (CRD_NAME, "toolpolicies.kars.azure.com", "karssandboxes.kars.azure.com"): + h = FixtureHarness() + h.existing = name + with self.subTest(name=name), self.assertRaisesRegex(AssertionError, "HTTP 200"): + self.bootstrap(h) + self.assertFalse(any(event[0] in ("create", "wait", "passed") for event in h.events)) + + def test_create_conflict_is_not_retried_as_apply_or_patch(self): + h = FixtureHarness() + h.create_failure = True + with self.assertRaisesRegex(AssertionError, "HTTP 409"): + self.bootstrap(h) + self.assertEqual(sum(event[0] == "create" for event in h.events), 1) + self.assertFalse(any(event[0] in ("wait", "passed") for event in h.events)) + + def test_established_failure_never_reaches_discovery_or_hooks(self): + h = FixtureHarness() + h.wait_failure = True + with self.assertRaisesRegex(AssertionError, "wait-timeout"): + self.bootstrap(h) + self.assertFalse(any(event[0] in ("poll", "passed") for event in h.events)) + + def test_discovery_retries_only_missing_group_or_resources(self): + h = FixtureHarness() + incomplete = discovery() + incomplete["resources"].pop() + h.responses = [(404, {"kind": "Status", "reason": "NotFound"}), + (200, incomplete), (200, discovery())] + self.bootstrap(h) + self.assertEqual(h.responses, []) + self.assertEqual(h.events[-1][0], "passed") + + def test_discovery_auth_server_errors_and_malformed_responses_are_fatal(self): + for response in ((403, {"kind": "Status", "reason": "Forbidden"}), + (401, {"kind": "Status", "reason": "Unauthorized"}), + (500, {"kind": "Status", "reason": "InternalError"}), + (404, {"kind": "Status", "reason": "Forbidden"}), + (200, {"kind": "Secret", "data": {"token": "must-not-log"}})): + h = FixtureHarness() + h.responses = [response, (200, discovery())] + with self.subTest(code=response[0]), self.assertRaises(AssertionError) as failure: + self.bootstrap(h) + self.assertNotIn("must-not-log", str(failure.exception)) + self.assertEqual(len(h.responses), 1) + self.assertFalse(any(event[0] == "passed" for event in h.events)) + + def test_wrong_discovered_identity_or_verbs_fail_instead_of_retrying(self): + for key, value in (("kind", "Secret"), ("namespaced", False), ("verbs", ["get"])): + h = FixtureHarness() + body = discovery() + body["resources"][0][key] = value + h.responses = [(200, body)] + with self.subTest(key=key), self.assertRaises(AssertionError): + self.bootstrap(h) + self.assertFalse(any(event[0] == "passed" for event in h.events)) + + def test_discovery_deadline_is_not_success(self): + h = FixtureHarness() + body = discovery() + body["resources"] = [] + h.responses = [(200, body)] * 3 + with self.assertRaisesRegex(AssertionError, "bounded discovery deadline"): + self.bootstrap(h) + self.assertFalse(any(event[0] == "passed" for event in h.events)) + + +if __name__ == "__main__": + unittest.main() From 7d4656f6c7ca3201e8ee0bc3646642aea4e41ad1 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 12:52:08 +0200 Subject: [PATCH 25/62] test(e2e): preserve Helm ownership in historical CRD bootstrap Use Helm's field manager for create-only legacy CRD setup, in addition to its release labels and annotations. Add a disposable same-Kind API A/B proof of kubectl-create versus Helm ownership through unchanged apply and a dry-run current-schema upgrade, with UID-fenced cleanup and allowlisted conflict reporting. Never force conflicts or adopt existing objects. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 5 +- tests/e2e/sre_authority/common.py | 1 + tests/e2e/sre_authority/legacy_crd_probe.py | 110 ++++++++++++++++++ tests/e2e/sre_authority/legacy_crds.py | 4 +- tests/e2e/sre_authority/legacy_crds_test.py | 15 ++- .../e2e/sre_authority/registration_schema.py | 4 +- 6 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/sre_authority/legacy_crd_probe.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 872c4291b..35be941fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,9 +416,11 @@ jobs: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Check public-schema diagnostic privacy - run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test + run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test - name: Create the same disposable API server as the real harness run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" + - name: Prove historical CRD bootstrap ownership permits unchanged Helm schema upgrades + run: PYTHONPATH=tests/e2e python3 -m sre_authority.legacy_crd_probe - name: Validate the SRE CRD against the actual API server id: sre_schema run: python3 tests/e2e/sre_authority/registration_schema.py --exercise @@ -438,6 +440,7 @@ jobs: name: sre-crd-schema-${{ github.run_id }} path: | e2e-sre-schema-diag/versions.json + e2e-sre-schema-diag/legacy-crd-ownership.json e2e-sre-schema-diag/validation.json e2e-sre-schema-diag/validation-instances.json e2e-sre-schema-diag/namespace-accessor-candidate.json diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 8deea8ff0..8f9aa0c38 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -73,6 +73,7 @@ def command_error_category(stderr): ("unknown flag", "cli-argument"), ("required value", "required-field"), ("no matches for kind", "api-discovery"), + ("conflict occurred while applying object", "server-side-apply-conflict"), ("the server doesn't have a resource type", "api-discovery"), ("timed out waiting", "wait-timeout")): if needle in text: diff --git a/tests/e2e/sre_authority/legacy_crd_probe.py b/tests/e2e/sre_authority/legacy_crd_probe.py new file mode 100644 index 000000000..65fed3c7e --- /dev/null +++ b/tests/e2e/sre_authority/legacy_crd_probe.py @@ -0,0 +1,110 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Same-Kind API proof of historical CRD field ownership; no workload execution.""" + +import copy +from pathlib import Path +import sys +import time + +from sre_authority.bootstrap_probe import converted_objects +from sre_authority.common import require +from sre_authority.fixtures import LEGACY_COMMIT +from sre_authority.legacy_crds import validate_historical_crd +from sre_authority.registration_schema import ( + CONTEXT, CRD_PATH, command, kind_proxy, request, write_report, +) + +NAME = "karssreactions.kars.azure.com" +PATH = f"{CRD_PATH}/{NAME}" +IDENTITY = ("karssreactions", "KarsSREAction", "Namespaced") + + +def conflict_report(code, body): + return {"httpStatus": code, "fieldManagerConflict": bool( + code == 409 and isinstance(body, dict) and body.get("kind") == "Status" + and body.get("reason") == "Conflict" + and any(cause.get("reason") == "FieldManagerConflict" + for cause in body.get("details", {}).get("causes", [])))} + + +def cleanup(port, uid): + code, current = request(port, "GET", PATH) + require(code == 200 and current.get("metadata", {}).get("uid") == uid, + "Historical CRD ownership fixture changed before cleanup") + code, _ = request(port, "DELETE", PATH, { + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}, + }) + require(code in (200, 202), "Historical CRD ownership cleanup failed") + end = time.monotonic() + 30 + while time.monotonic() < end: + code, _ = request(port, "GET", PATH) + if code == 404: + return + require(code == 200, "Historical CRD cleanup inspection failed") + time.sleep(0.2) + raise AssertionError("Historical CRD ownership cleanup timed out") + + +def exercise(root): + command("legacy-fetch", ["git", "fetch", "--no-tags", "--depth=1", + "https://github.com/Azure/kars.git", LEGACY_COMMIT], root=root) + historical = command("legacy-source", ["git", "show", + f"{LEGACY_COMMIT}:deploy/helm/kars/templates/crd-karssreaction.yaml"], root=root) + current = command("current-render", ["helm", "template", "kars", str(root / "deploy/helm/kars"), + "--namespace", "kars-system", "--show-only", "templates/crd-karssreaction.yaml"], root=root) + objects = [] + for rendered in (historical, current): + parsed = converted_objects(command("legacy-conversion", [ + "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", + "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", + ], root=root, data=rendered)) + require(len(parsed) == 1, "Historical field ownership proof requires one public CRD") + validate_historical_crd(parsed[0], IDENTITY) + objects.append(parsed[0]) + old, new = objects + with kind_proxy(root) as (port, version): + reports = [] + for manager, expected in (("kubectl-create", 409), ("helm", 200)): + code, _ = request(port, "GET", PATH) + require(code == 404, "Historical field ownership proof refuses an existing CRD") + desired = copy.deepcopy(old) + desired["metadata"]["labels"]["app.kubernetes.io/managed-by"] = "Helm" + desired["metadata"]["annotations"] = { + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": "kars-system"} + code, created = request(port, "POST", CRD_PATH + f"?fieldManager={manager}", desired) + require(code == 201 and created.get("metadata", {}).get("uid"), + "Historical ownership fixture CREATE failed") + uid = created["metadata"]["uid"] + try: + code, _ = request(port, "PATCH", PATH + "?fieldManager=helm&force=false", desired, + content_type="application/apply-patch+yaml") + require(code == 200, "Unchanged historical Helm apply failed") + updated = copy.deepcopy(new) + updated["metadata"] = desired["metadata"] + code, body = request(port, "PATCH", PATH + "?fieldManager=helm&force=false&dryRun=All", updated, + content_type="application/apply-patch+yaml") + report = {"bootstrapManager": manager, "expectedStatus": expected, **conflict_report(code, body)} + reports.append(report) + write_report(root, "legacy-crd-ownership.json", { + "apiServer": version, "legacyCommit": LEGACY_COMMIT, "resource": NAME, "cases": reports}) + require(code == expected and (code != 409 or report["fieldManagerConflict"]), + "Historical Helm field ownership did not match the expected API result") + finally: + failed = sys.exc_info()[0] is not None + try: + cleanup(port, uid) + except Exception: + if not failed: + raise + print("SRE-CRD-OWNERSHIP-CLEANUP-FAIL", flush=True) + + +if __name__ == "__main__": + try: + exercise(Path(__file__).resolve().parents[3]) + except Exception as error: + print(f"SRE-CRD-OWNERSHIP-FAIL category={type(error).__name__}", flush=True) + raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/legacy_crds.py b/tests/e2e/sre_authority/legacy_crds.py index 0fbd6f0cd..c05b1bc41 100644 --- a/tests/e2e/sre_authority/legacy_crds.py +++ b/tests/e2e/sre_authority/legacy_crds.py @@ -123,7 +123,9 @@ def bootstrap_legacy_crds(h, chart, sources): obj["metadata"]["annotations"] = { "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM, } - h.create(obj) + # Use Helm's own field-manager name as well as its release metadata. + # Otherwise its later SSA schema upgrade conflicts with kubectl-create. + h.create(obj, manager="helm") h.k("wait", "--for=condition=Established", *[f'crd/{obj["metadata"]["name"]}' for obj in objects], "--timeout=60s", timeout=70) diff --git a/tests/e2e/sre_authority/legacy_crds_test.py b/tests/e2e/sre_authority/legacy_crds_test.py index 275917872..13c934eef 100644 --- a/tests/e2e/sre_authority/legacy_crds_test.py +++ b/tests/e2e/sre_authority/legacy_crds_test.py @@ -13,6 +13,7 @@ CRDS, GROUP_VERSION, IDENTITIES, bootstrap_legacy_crds, render_legacy_crds, validate_rendered_crds, ) from sre_authority.registration_schema import CRD_NAME +from sre_authority.legacy_crd_probe import conflict_report def historical_objects(): @@ -61,8 +62,8 @@ def api(self, method, path, *, status): raise AssertionError(f"HTTP {code}") return types.SimpleNamespace(status_code=code, json=lambda: body) - def create(self, obj): - self.events.append(("create", obj)) + def create(self, obj, *, manager): + self.events.append(("create", obj, manager)) if self.create_failure: raise AssertionError("HTTP 409") @@ -156,6 +157,7 @@ def test_create_only_ownership_and_all_readiness_precede_hooks(self): [CRD_NAME] + names) creates = [event[1] for event in h.events if event[0] == "create"] self.assertEqual(len(creates), 18) + self.assertTrue(all(event[2] == "helm" for event in h.events if event[0] == "create")) for original, created in zip(originals, creates): self.assertNotIn("annotations", original["metadata"]) self.assertEqual(created["spec"], original["spec"]) @@ -239,6 +241,15 @@ def test_discovery_deadline_is_not_success(self): self.bootstrap(h) self.assertFalse(any(event[0] == "passed" for event in h.events)) + def test_field_manager_evidence_requires_actual_conflict_without_echoing_bodies(self): + body = {"kind": "Status", "reason": "Conflict", "message": "must-not-log", + "details": {"causes": [{"reason": "FieldManagerConflict", "message": "must-not-log"}]}} + self.assertEqual(conflict_report(409, body), {"httpStatus": 409, "fieldManagerConflict": True}) + for code, invalid in ((200, body), (403, body), (409, {}), (409, None), + (409, {**body, "reason": "Forbidden"})): + self.assertFalse(conflict_report(code, invalid)["fieldManagerConflict"]) + self.assertNotIn("must-not-log", str(conflict_report(code, invalid))) + if __name__ == "__main__": unittest.main() diff --git a/tests/e2e/sre_authority/registration_schema.py b/tests/e2e/sre_authority/registration_schema.py index 2b1f3326b..d2ba8d407 100644 --- a/tests/e2e/sre_authority/registration_schema.py +++ b/tests/e2e/sre_authority/registration_schema.py @@ -111,10 +111,10 @@ def command(stage, args, *, root, data=None): return result.stdout -def request(port, method, path, obj=None): +def request(port, method, path, obj=None, *, content_type="application/json"): body = None if obj is None else json.dumps(obj).encode() req = Request(f"http://127.0.0.1:{port}{path}", data=body, method=method, - headers={"Content-Type": "application/json", "Accept": "application/json"}) + headers={"Content-Type": content_type, "Accept": "application/json"}) opener = build_opener(ProxyHandler({})) try: response = opener.open(req, timeout=15) From 83850fadda1dc15b209bd66e0f048dcd51130ff1 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 13:01:45 +0200 Subject: [PATCH 26/62] test(e2e): let Helm own historical CRD creation and readiness The same-Kind A/B demonstrated FieldManagerConflict for POST-created historical CRDs with both kubectl-create and helm managers. Keep archive identity/absence validation read-only and use the existing versioned legacy waiter for initial Helm install; Helm waits on templated CRDs before post-install hooks and retains native Apply ownership. Prove the actual historical install, hook and current-authority server dry-run in the disposable API job, then reset its cluster before existing policy qualification. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 8 +- tests/e2e/sre_authority/fixtures.py | 24 +- tests/e2e/sre_authority/legacy_crd_probe.py | 127 ++++------- tests/e2e/sre_authority/legacy_crds.py | 76 ++----- tests/e2e/sre_authority/legacy_crds_test.py | 205 ++++-------------- .../e2e/sre_authority/registration_schema.py | 4 +- 6 files changed, 126 insertions(+), 318 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35be941fa..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -419,8 +419,12 @@ jobs: run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test - name: Create the same disposable API server as the real harness run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" - - name: Prove historical CRD bootstrap ownership permits unchanged Helm schema upgrades + - name: Prove native historical Helm wait orders CRDs before hooks and permits schema upgrades run: PYTHONPATH=tests/e2e python3 -m sre_authority.legacy_crd_probe + - name: Reset disposable cluster after historical Helm proof + run: | + kind delete cluster --name kars-e2e + kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG" - name: Validate the SRE CRD against the actual API server id: sre_schema run: python3 tests/e2e/sre_authority/registration_schema.py --exercise @@ -440,7 +444,7 @@ jobs: name: sre-crd-schema-${{ github.run_id }} path: | e2e-sre-schema-diag/versions.json - e2e-sre-schema-diag/legacy-crd-ownership.json + e2e-sre-schema-diag/legacy-helm-readiness.json e2e-sre-schema-diag/validation.json e2e-sre-schema-diag/validation-instances.json e2e-sre-schema-diag/namespace-accessor-candidate.json diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py index b9d8e2392..58a41535b 100644 --- a/tests/e2e/sre_authority/fixtures.py +++ b/tests/e2e/sre_authority/fixtures.py @@ -11,7 +11,7 @@ enrollment_json, fingerprint, require, ) from .credential_paths import seed_privacy_gaps -from .legacy_crds import bootstrap_legacy_crds +from .legacy_crds import preflight_legacy_crds from .registration_schema import create_registration_crd LEGACY_COMMIT = "8b206065608593667a40665b3f48225ef9ce278d" @@ -49,10 +49,7 @@ def role_binding(h, name, role, account, namespace=None, account_namespace=OPERA "subjects": [{"kind": "ServiceAccount", "name": account, "namespace": account_namespace}]}) -def prepare_legacy(h): - require(not h.get("validatingadmissionpolicy", "kars-sre-source-authority"), - "Legacy fixtures must precede the new SRE admission guards") - require(not h.get("namespace", RUNTIME), "Disposable cluster contains a pre-existing SRE namespace") +def install_historical_chart(h): h.run(["git", "fetch", "--no-tags", "--depth=1", "https://github.com/Azure/kars.git", LEGACY_COMMIT], timeout=90) # git archive is binary; invoke separately without decoding tar as text. import subprocess @@ -73,11 +70,15 @@ def prepare_legacy(h): for member in chart.getmembers() if member.isfile() and member.name.startswith("deploy/helm/kars/templates/crd") } - # The historical chart puts CRDs in templates/, so Helm's CRD-directory - # readiness does not order its post-install ToolPolicy hook. - bootstrap_legacy_crds(h, destination / "deploy/helm/kars", sources) + preflight_legacy_crds(h, destination / "deploy/helm/kars", sources) + version = h.run(["helm", "version", "--template", "{{.Version}}"]).strip() + wait_arg = h.run(["bash", "-c", 'source "$1"; sre_migration_helm_wait_arg "$2"', + "legacy-helm-wait", str(h.root / "tests/e2e/sre-authority.sh"), version]).strip() + # Helm's legacy waiter includes templated CRDs (Established), and runs + # before post-install hooks. Let Helm retain its native SSA ownership. h.run(["helm", "install", "kars", str(destination / "deploy/helm/kars"), "--kube-context", CONTEXT, "--namespace", SYSTEM, "--create-namespace", + wait_arg, "--timeout", "120s", "--set", "controller.replicas=0", "--set", "controller.image.repository=kars-controller", "--set", "controller.image.tag=e2e", "--set", "controller.image.pullPolicy=Never", "--set", "inferenceRouter.image.repository=kars-inference-router", @@ -91,6 +92,13 @@ def prepare_legacy(h): h.k("wait", "--for=condition=Established", "crd/karssandboxes.kars.azure.com", "--timeout=60s", timeout=70) h.run(["helm", "upgrade", "kars", str(destination / "deploy/helm/kars"), "--kube-context", CONTEXT, "--namespace", SYSTEM, "--reuse-values", "--set", "sre.enabled=true"], timeout=120) + + +def prepare_legacy(h): + require(not h.get("validatingadmissionpolicy", "kars-sre-source-authority"), + "Legacy fixtures must precede the new SRE admission guards") + require(not h.get("namespace", RUNTIME), "Disposable cluster contains a pre-existing SRE namespace") + install_historical_chart(h) source = h.get("karssandbox", "sre", SYSTEM) require(source is not None, "Historical chart did not create its source CR") h.state["legacy_source_uid"] = source["metadata"]["uid"] diff --git a/tests/e2e/sre_authority/legacy_crd_probe.py b/tests/e2e/sre_authority/legacy_crd_probe.py index 65fed3c7e..c5fc5589f 100644 --- a/tests/e2e/sre_authority/legacy_crd_probe.py +++ b/tests/e2e/sre_authority/legacy_crd_probe.py @@ -1,110 +1,59 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Same-Kind API proof of historical CRD field ownership; no workload execution.""" +"""Same-Kind historical Helm wait/upgrade proof without controller execution.""" -import copy from pathlib import Path -import sys import time +import types from sre_authority.bootstrap_probe import converted_objects -from sre_authority.common import require -from sre_authority.fixtures import LEGACY_COMMIT -from sre_authority.legacy_crds import validate_historical_crd +from sre_authority.common import CONTEXT, Harness, SYSTEM, require +from sre_authority.fixtures import LEGACY_COMMIT, install_historical_chart from sre_authority.registration_schema import ( - CONTEXT, CRD_PATH, command, kind_proxy, request, write_report, + create_registration_crd, kind_proxy, request, write_report, ) -NAME = "karssreactions.kars.azure.com" -PATH = f"{CRD_PATH}/{NAME}" -IDENTITY = ("karssreactions", "KarsSREAction", "Namespaced") - - -def conflict_report(code, body): - return {"httpStatus": code, "fieldManagerConflict": bool( - code == 409 and isinstance(body, dict) and body.get("kind") == "Status" - and body.get("reason") == "Conflict" - and any(cause.get("reason") == "FieldManagerConflict" - for cause in body.get("details", {}).get("causes", [])))} - - -def cleanup(port, uid): - code, current = request(port, "GET", PATH) - require(code == 200 and current.get("metadata", {}).get("uid") == uid, - "Historical CRD ownership fixture changed before cleanup") - code, _ = request(port, "DELETE", PATH, { - "apiVersion": "v1", "kind": "DeleteOptions", - "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}, - }) - require(code in (200, 202), "Historical CRD ownership cleanup failed") - end = time.monotonic() + 30 - while time.monotonic() < end: - code, _ = request(port, "GET", PATH) - if code == 404: - return - require(code == 200, "Historical CRD cleanup inspection failed") - time.sleep(0.2) - raise AssertionError("Historical CRD ownership cleanup timed out") - def exercise(root): - command("legacy-fetch", ["git", "fetch", "--no-tags", "--depth=1", - "https://github.com/Azure/kars.git", LEGACY_COMMIT], root=root) - historical = command("legacy-source", ["git", "show", - f"{LEGACY_COMMIT}:deploy/helm/kars/templates/crd-karssreaction.yaml"], root=root) - current = command("current-render", ["helm", "template", "kars", str(root / "deploy/helm/kars"), - "--namespace", "kars-system", "--show-only", "templates/crd-karssreaction.yaml"], root=root) - objects = [] - for rendered in (historical, current): - parsed = converted_objects(command("legacy-conversion", [ - "kubectl", "--context", CONTEXT, "--request-timeout=15s", "create", - "--dry-run=client", "--validate=strict", "-f", "-", "-o", "json", - ], root=root, data=rendered)) - require(len(parsed) == 1, "Historical field ownership proof requires one public CRD") - validate_historical_crd(parsed[0], IDENTITY) - objects.append(parsed[0]) - old, new = objects + h = Harness.__new__(Harness) + h.root, h.work = root, root / ".e2e-legacy-helm" + h.work.mkdir(mode=0o700) + h.deadline, h.phase = time.monotonic() + 300, "legacy-helm-proof" with kind_proxy(root) as (port, version): - reports = [] - for manager, expected in (("kubectl-create", 409), ("helm", 200)): - code, _ = request(port, "GET", PATH) - require(code == 404, "Historical field ownership proof refuses an existing CRD") - desired = copy.deepcopy(old) - desired["metadata"]["labels"]["app.kubernetes.io/managed-by"] = "Helm" - desired["metadata"]["annotations"] = { - "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": "kars-system"} - code, created = request(port, "POST", CRD_PATH + f"?fieldManager={manager}", desired) - require(code == 201 and created.get("metadata", {}).get("uid"), - "Historical ownership fixture CREATE failed") - uid = created["metadata"]["uid"] - try: - code, _ = request(port, "PATCH", PATH + "?fieldManager=helm&force=false", desired, - content_type="application/apply-patch+yaml") - require(code == 200, "Unchanged historical Helm apply failed") - updated = copy.deepcopy(new) - updated["metadata"] = desired["metadata"] - code, body = request(port, "PATCH", PATH + "?fieldManager=helm&force=false&dryRun=All", updated, - content_type="application/apply-patch+yaml") - report = {"bootstrapManager": manager, "expectedStatus": expected, **conflict_report(code, body)} - reports.append(report) - write_report(root, "legacy-crd-ownership.json", { - "apiServer": version, "legacyCommit": LEGACY_COMMIT, "resource": NAME, "cases": reports}) - require(code == expected and (code != 409 or report["fieldManagerConflict"]), - "Historical Helm field ownership did not match the expected API result") - finally: - failed = sys.exc_info()[0] is not None - try: - cleanup(port, uid) - except Exception: - if not failed: - raise - print("SRE-CRD-OWNERSHIP-CLEANUP-FAIL", flush=True) + def api(method, path, *, body=None, status=None): + code, obj = request(port, method, path, body) + if status is not None: + require(code in (status if isinstance(status, tuple) else (status,)), + f"Historical Helm API proof: HTTP {code}") + return types.SimpleNamespace(status_code=code, json=lambda: obj) + h.api = api + install_historical_chart(h) + rendered = h.run(["helm", "template", "kars", str(root / "deploy/helm/kars"), + "--namespace", SYSTEM, "--show-only", "templates/crd-karssreregistration.yaml"]) + objects = converted_objects(h.k("create", "--dry-run=client", "--validate=strict", + "-f", "-", "-o", "json", data=rendered)) + require(len(objects) == 1, "Historical Helm proof requires one registration CRD") + obj = objects[0] + obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" + obj["metadata"]["annotations"] = { + "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM} + create_registration_crd(h, obj) + h.k("wait", "--for=condition=Established", "crd/karssreregistrations.kars.azure.com", + "--timeout=60s", timeout=70) + h.run(["helm", "--kube-context", CONTEXT, "upgrade", "kars", str(root / "deploy/helm/kars"), + "--namespace", SYSTEM, "--reset-then-reuse-values", "--set", "sre.authorityStage=true", + "--dry-run=server"], timeout=90) + write_report(root, "legacy-helm-readiness.json", { + "apiServer": version, "legacyCommit": LEGACY_COMMIT, + "historicalInstallAndPostInstallHook": "passed", "currentAuthorityServerDryRun": "passed", + "controllerReplicas": 0, "legacyCRDs": 18, "crdCreation": "native-Helm-only"}) if __name__ == "__main__": try: exercise(Path(__file__).resolve().parents[3]) except Exception as error: - print(f"SRE-CRD-OWNERSHIP-FAIL category={type(error).__name__}", flush=True) + detail = str(error) if isinstance(error, AssertionError) else type(error).__name__ + print(f"SRE-LEGACY-HELM-FAIL {detail}", flush=True) raise SystemExit(1) from None diff --git a/tests/e2e/sre_authority/legacy_crds.py b/tests/e2e/sre_authority/legacy_crds.py index c05b1bc41..1afc669c7 100644 --- a/tests/e2e/sre_authority/legacy_crds.py +++ b/tests/e2e/sre_authority/legacy_crds.py @@ -1,11 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Create-only readiness for CRDs from the immutable historical Helm fixture.""" - -import copy -import json +"""Read-only validation of CRDs from the immutable historical Helm fixture.""" +from .bootstrap_probe import converted_objects from .common import SYSTEM, require from .registration_schema import CRD_NAME, CRD_PATH @@ -31,7 +29,6 @@ } # The historical crd.yaml contains both KarsSandbox and KarsPairing. IDENTITIES = (*CRDS.values(), ("karspairings", "KarsPairing", "Namespaced")) -GROUP_VERSION = "kars.azure.com/v1alpha1" def validate_rendered_crds(rendered, historical): @@ -89,65 +86,26 @@ def render_legacy_crds(h, chart, sources): for filename in CRDS: args += ["--show-only", f"templates/{filename}"] rendered = h.run(args, timeout=45) - converter = """ -const y=require('node:module').createRequire(process.cwd()+'/cli/package.json')('yaml'); -const input=JSON.parse(require('fs').readFileSync(0,'utf8')); -function documents(text) { - return y.parseAllDocuments(text).map(doc => { - if (doc.errors.length) throw Error('Invalid historical YAML'); - return doc.toJSON(); - }).filter(doc => doc !== null); -} -const historical=Object.fromEntries(Object.entries(input.sources).map(([name,text]) => { - return [name,documents(text)]; -})); -console.log(JSON.stringify({historical,rendered:documents(input.rendered)})); -""" - parsed = json.loads(h.run(["node", "-e", converter], - data=json.dumps({"sources": sources, "rendered": rendered}), timeout=20)) - return validate_rendered_crds(parsed["rendered"], parsed["historical"]) + def convert(text): + return converted_objects(h.k("create", "--dry-run=client", "--validate=strict", + "-f", "-", "-o", "json", data=text)) + original = convert("\n---\n".join(sources[filename] for filename in CRDS)) + require(len(original) == len(IDENTITIES), "Immutable historical CRD count changed") + historical, offset = {}, 0 + for filename in CRDS: + count = 2 if filename == "crd.yaml" else 1 + historical[filename] = original[offset:offset + count] + offset += count + return validate_rendered_crds(convert(rendered), historical) -def bootstrap_legacy_crds(h, chart, sources): +def preflight_legacy_crds(h, chart, sources): objects = render_legacy_crds(h, chart, sources) - # Preflight every absence before the first CREATE. An existing object, - # including one claiming this release, is never adopted or patched. + # Only Helm creates these objects, retaining its native SSA field ownership. + # An existing object, even one claiming this release, is never adopted. for name in [CRD_NAME] + [obj["metadata"]["name"] for obj in objects]: response = h.api("GET", f"{CRD_PATH}/{name}", status=404) body = response.json() require(body.get("kind") == "Status" and body.get("reason") == "NotFound", "Historical CRD absence was not a Kubernetes NotFound") - for original in objects: - obj = copy.deepcopy(original) - obj["metadata"].setdefault("labels", {})["app.kubernetes.io/managed-by"] = "Helm" - obj["metadata"]["annotations"] = { - "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": SYSTEM, - } - # Use Helm's own field-manager name as well as its release metadata. - # Otherwise its later SSA schema upgrade conflicts with kubectl-create. - h.create(obj, manager="helm") - h.k("wait", "--for=condition=Established", - *[f'crd/{obj["metadata"]["name"]}' for obj in objects], "--timeout=60s", timeout=70) - - def discovered(): - response = h.api("GET", f"/apis/{GROUP_VERSION}", status=(200, 404)) - body = response.json() - if response.status_code == 404: - require(body.get("kind") == "Status" and body.get("reason") == "NotFound", - "Historical API discovery returned an unexpected response") - return False - require(body.get("kind") == "APIResourceList" and body.get("groupVersion") == GROUP_VERSION - and isinstance(body.get("resources"), list), - "Historical API discovery returned an unexpected resource list") - resources = {item["name"]: item for item in body["resources"]} - for plural, kind, scope in IDENTITIES: - if plural not in resources: - return False - item = resources[plural] - require(item.get("kind") == kind and item.get("namespaced") == (scope == "Namespaced") - and {"create", "get", "list", "watch"}.issubset(item.get("verbs", [])), - "Historical API discovery differs from the immutable CRD identity") - return True - - h.poll("Historical CRD API discovery before Helm hooks", discovered, seconds=60) - h.passed("All 18 unchanged historical CRDs created with Helm ownership, Established and discovered before initial hooks") + h.passed("All 18 historical CRDs match the immutable archive and are absent before Helm-owned creation") diff --git a/tests/e2e/sre_authority/legacy_crds_test.py b/tests/e2e/sre_authority/legacy_crds_test.py index 13c934eef..41ec4bffc 100644 --- a/tests/e2e/sre_authority/legacy_crds_test.py +++ b/tests/e2e/sre_authority/legacy_crds_test.py @@ -10,10 +10,9 @@ from unittest.mock import Mock, patch from sre_authority.legacy_crds import ( - CRDS, GROUP_VERSION, IDENTITIES, bootstrap_legacy_crds, render_legacy_crds, validate_rendered_crds, + CRDS, IDENTITIES, preflight_legacy_crds, render_legacy_crds, validate_rendered_crds, ) from sre_authority.registration_schema import CRD_NAME -from sre_authority.legacy_crd_probe import conflict_report def historical_objects(): @@ -34,62 +33,7 @@ def flattened(historical): return [obj for objects in historical.values() for obj in objects] -def discovery(): - return {"kind": "APIResourceList", "groupVersion": GROUP_VERSION, "resources": [ - {"name": plural, "kind": kind, "namespaced": scope == "Namespaced", - "verbs": ["create", "get", "list", "watch"]} - for plural, kind, scope in IDENTITIES - ]} - - -class FixtureHarness: - def __init__(self): - self.events = [] - self.responses = [(200, discovery())] - self.existing = None - self.create_failure = False - self.wait_failure = False - - def api(self, method, path, *, status): - self.events.append(("api", method, path)) - if path == f"/apis/{GROUP_VERSION}": - code, body = self.responses.pop(0) - elif path.endswith("/" + str(self.existing)): - code, body = 200, {"kind": "CustomResourceDefinition"} - else: - code, body = 404, {"kind": "Status", "reason": "NotFound"} - if code not in (status if isinstance(status, tuple) else (status,)): - raise AssertionError(f"HTTP {code}") - return types.SimpleNamespace(status_code=code, json=lambda: body) - - def create(self, obj, *, manager): - self.events.append(("create", obj, manager)) - if self.create_failure: - raise AssertionError("HTTP 409") - - def k(self, *args, **kwargs): - self.events.append(("wait", args, kwargs)) - if self.wait_failure: - raise AssertionError("wait-timeout") - - def poll(self, label, predicate, *, seconds): - self.events.append(("poll", label, seconds)) - for _ in range(3): - if predicate(): - return True - raise AssertionError("bounded discovery deadline") - - def passed(self, message): - self.events.append(("passed", message)) - - class LegacyCRDTests(unittest.TestCase): - def bootstrap(self, h): - objects = flattened(historical_objects()) - with patch("sre_authority.legacy_crds.render_legacy_crds", return_value=objects): - bootstrap_legacy_crds(h, Path("unused-test-chart"), {}) - return objects - def test_render_requires_exact_historical_content_not_just_a_matching_name(self): historical = historical_objects() rendered = flattened(copy.deepcopy(historical)) @@ -130,7 +74,6 @@ def changed(section, key, value): def test_changed_extracted_source_and_unexpected_inventory_fail_before_render(self): h = Mock() - # A small path double avoids filesystem writes in these pure checks. class Chart: def __truediv__(self, _name): return self @@ -146,110 +89,56 @@ def read_text(self): with self.assertRaises(AssertionError): render_legacy_crds(h, Chart(), sources) h.run.assert_not_called() + h.k.assert_not_called() - def test_create_only_ownership_and_all_readiness_precede_hooks(self): - h = FixtureHarness() - originals = self.bootstrap(h) - names = [obj["metadata"]["name"] for obj in originals] - self.assertEqual(len(names), 18) - self.assertNotIn(CRD_NAME, names) - self.assertEqual([event[2].rsplit("/", 1)[-1] for event in h.events[:19]], - [CRD_NAME] + names) - creates = [event[1] for event in h.events if event[0] == "create"] - self.assertEqual(len(creates), 18) - self.assertTrue(all(event[2] == "helm" for event in h.events if event[0] == "create")) - for original, created in zip(originals, creates): - self.assertNotIn("annotations", original["metadata"]) - self.assertEqual(created["spec"], original["spec"]) - self.assertEqual(created["metadata"]["labels"]["app.kubernetes.io/managed-by"], "Helm") - self.assertEqual(created["metadata"]["annotations"], { - "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": "kars-system"}) - wait = next(event for event in h.events if event[0] == "wait") - self.assertEqual(wait[1], ("wait", "--for=condition=Established", - *[f"crd/{name}" for name in names], "--timeout=60s")) - self.assertLess(h.events.index(wait), next(i for i, e in enumerate(h.events) if e[0] == "poll")) - self.assertEqual(h.events[-1][0], "passed") - fixture = (Path(__file__).with_name("fixtures.py")).read_text() - self.assertLess(fixture.index("bootstrap_legacy_crds(h,"), fixture.index('h.run(["helm", "install"')) - self.assertLess(fixture.index('h.run(["helm", "install"'), fixture.index('h.get("toolpolicy"')) + def test_every_absence_is_read_only_before_native_helm_creates_any_crd(self): + h = Mock() + h.api.return_value = types.SimpleNamespace(json=lambda: {"kind": "Status", "reason": "NotFound"}) + objects = flattened(historical_objects()) + with patch("sre_authority.legacy_crds.render_legacy_crds", return_value=objects): + preflight_legacy_crds(h, Path("unused-test-chart"), {}) + names = [CRD_NAME] + [obj["metadata"]["name"] for obj in objects] + self.assertEqual(len(names), 19) + self.assertEqual([call.args[1].rsplit("/", 1)[-1] for call in h.api.call_args_list], names) + self.assertTrue(all(call.args[0] == "GET" and call.kwargs == {"status": 404} + for call in h.api.call_args_list)) + h.create.assert_not_called() + h.k.assert_not_called() + h.passed.assert_called_once() + + def test_existing_or_inaccessible_crd_aborts_without_adoption_or_retry(self): + for code in (200, 401, 403, 409, 500): + h = Mock() + h.api.side_effect = AssertionError(f"HTTP {code}") + with patch("sre_authority.legacy_crds.render_legacy_crds", + return_value=flattened(historical_objects())): + with self.subTest(code=code), self.assertRaisesRegex(AssertionError, f"HTTP {code}"): + preflight_legacy_crds(h, Path("unused-test-chart"), {}) + h.api.assert_called_once() + h.create.assert_not_called() + h.passed.assert_not_called() + + def test_false_not_found_is_not_absence_proof(self): + for body in ({"kind": "Secret", "reason": "NotFound"}, {"kind": "Status", "reason": "Forbidden"}): + h = Mock() + h.api.return_value = types.SimpleNamespace(json=lambda: body) + with patch("sre_authority.legacy_crds.render_legacy_crds", + return_value=flattened(historical_objects())), self.assertRaises(AssertionError): + preflight_legacy_crds(h, Path("unused-test-chart"), {}) + h.create.assert_not_called() + h.passed.assert_not_called() + + def test_initial_helm_uses_existing_versioned_waiter_without_custom_creation(self): + fixture = Path(__file__).with_name("fixtures.py").read_text() + install = fixture.split("def install_historical_chart(h):", 1)[1].split("def prepare_legacy(h):", 1)[0] + self.assertLess(install.index("preflight_legacy_crds(h,"), install.index('h.run(["helm", "install"')) + self.assertIn('sre_migration_helm_wait_arg "$2"', install) + self.assertIn('wait_arg, "--timeout", "120s"', install) + self.assertLess(install.index('h.run(["helm", "install"'), install.index('h.get("toolpolicy"')) + self.assertNotIn("h.create(", install) for bypass in ("--take-ownership", "--force", "--validate=false", "--no-hooks", "governance.enabled=false"): self.assertNotIn(bypass, fixture) - def test_existing_foreign_or_even_release_named_crd_is_never_adopted(self): - for name in (CRD_NAME, "toolpolicies.kars.azure.com", "karssandboxes.kars.azure.com"): - h = FixtureHarness() - h.existing = name - with self.subTest(name=name), self.assertRaisesRegex(AssertionError, "HTTP 200"): - self.bootstrap(h) - self.assertFalse(any(event[0] in ("create", "wait", "passed") for event in h.events)) - - def test_create_conflict_is_not_retried_as_apply_or_patch(self): - h = FixtureHarness() - h.create_failure = True - with self.assertRaisesRegex(AssertionError, "HTTP 409"): - self.bootstrap(h) - self.assertEqual(sum(event[0] == "create" for event in h.events), 1) - self.assertFalse(any(event[0] in ("wait", "passed") for event in h.events)) - - def test_established_failure_never_reaches_discovery_or_hooks(self): - h = FixtureHarness() - h.wait_failure = True - with self.assertRaisesRegex(AssertionError, "wait-timeout"): - self.bootstrap(h) - self.assertFalse(any(event[0] in ("poll", "passed") for event in h.events)) - - def test_discovery_retries_only_missing_group_or_resources(self): - h = FixtureHarness() - incomplete = discovery() - incomplete["resources"].pop() - h.responses = [(404, {"kind": "Status", "reason": "NotFound"}), - (200, incomplete), (200, discovery())] - self.bootstrap(h) - self.assertEqual(h.responses, []) - self.assertEqual(h.events[-1][0], "passed") - - def test_discovery_auth_server_errors_and_malformed_responses_are_fatal(self): - for response in ((403, {"kind": "Status", "reason": "Forbidden"}), - (401, {"kind": "Status", "reason": "Unauthorized"}), - (500, {"kind": "Status", "reason": "InternalError"}), - (404, {"kind": "Status", "reason": "Forbidden"}), - (200, {"kind": "Secret", "data": {"token": "must-not-log"}})): - h = FixtureHarness() - h.responses = [response, (200, discovery())] - with self.subTest(code=response[0]), self.assertRaises(AssertionError) as failure: - self.bootstrap(h) - self.assertNotIn("must-not-log", str(failure.exception)) - self.assertEqual(len(h.responses), 1) - self.assertFalse(any(event[0] == "passed" for event in h.events)) - - def test_wrong_discovered_identity_or_verbs_fail_instead_of_retrying(self): - for key, value in (("kind", "Secret"), ("namespaced", False), ("verbs", ["get"])): - h = FixtureHarness() - body = discovery() - body["resources"][0][key] = value - h.responses = [(200, body)] - with self.subTest(key=key), self.assertRaises(AssertionError): - self.bootstrap(h) - self.assertFalse(any(event[0] == "passed" for event in h.events)) - - def test_discovery_deadline_is_not_success(self): - h = FixtureHarness() - body = discovery() - body["resources"] = [] - h.responses = [(200, body)] * 3 - with self.assertRaisesRegex(AssertionError, "bounded discovery deadline"): - self.bootstrap(h) - self.assertFalse(any(event[0] == "passed" for event in h.events)) - - def test_field_manager_evidence_requires_actual_conflict_without_echoing_bodies(self): - body = {"kind": "Status", "reason": "Conflict", "message": "must-not-log", - "details": {"causes": [{"reason": "FieldManagerConflict", "message": "must-not-log"}]}} - self.assertEqual(conflict_report(409, body), {"httpStatus": 409, "fieldManagerConflict": True}) - for code, invalid in ((200, body), (403, body), (409, {}), (409, None), - (409, {**body, "reason": "Forbidden"})): - self.assertFalse(conflict_report(code, invalid)["fieldManagerConflict"]) - self.assertNotIn("must-not-log", str(conflict_report(code, invalid))) - if __name__ == "__main__": unittest.main() diff --git a/tests/e2e/sre_authority/registration_schema.py b/tests/e2e/sre_authority/registration_schema.py index d2ba8d407..2b1f3326b 100644 --- a/tests/e2e/sre_authority/registration_schema.py +++ b/tests/e2e/sre_authority/registration_schema.py @@ -111,10 +111,10 @@ def command(stage, args, *, root, data=None): return result.stdout -def request(port, method, path, obj=None, *, content_type="application/json"): +def request(port, method, path, obj=None): body = None if obj is None else json.dumps(obj).encode() req = Request(f"http://127.0.0.1:{port}{path}", data=body, method=method, - headers={"Content-Type": content_type, "Accept": "application/json"}) + headers={"Content-Type": "application/json", "Accept": "application/json"}) opener = build_opener(ProxyHandler({})) try: response = opener.open(req, timeout=15) From 0137a05fb642f3ce93946998a55ef24a29934ce9 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 13:42:09 +0200 Subject: [PATCH 27/62] test(e2e): diagnose built-in controller private ReplicaSet admission Real legacy migration now reaches Ready with denial-before-issuance and rotation proof, but subsequent SRE install times out with ReplicaSetCreateError. Capture only allowlisted policy facts from runtime status and add same-Kind dry-run ordinary/private ReplicaSet cases under the existing Deployment controller principal, paired with actual cluster-scope authorization reviews. Preserve all production policies and RBAC unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/bootstrap_cases.py | 48 +++++++++++++++++- tests/e2e/sre_authority/bootstrap_probe.py | 5 ++ .../e2e/sre_authority/bootstrap_probe_test.py | 50 +++++++++++++++++++ tests/e2e/sre_authority/common.py | 5 +- 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index ebd93a45a..04422340a 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -13,12 +13,13 @@ from .registration_schema import request USER = "system:serviceaccount:e2e-sre-bootstrap:tenant" +DEPLOYMENT_CONTROLLER = "system:serviceaccount:kube-system:deployment-controller" -def as_tenant(port, path, obj): +def as_tenant(port, path, obj, *, user=USER): req = Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(obj).encode(), method="POST", headers={"Content-Type": "application/json", "Accept": "application/json", - "Impersonate-User": USER}) + "Impersonate-User": user}) try: response = build_opener(ProxyHandler({})).open(req, timeout=15) except HTTPError as error: @@ -95,3 +96,46 @@ def admission_cases(port, policies): result["matched"] = result["matched"] and result["paramsPreserved"] reports.append(result) return reports + + +def deployment_controller_cases(port, policies): + code, account = request(port, "GET", "/api/v1/namespaces/kube-system/serviceaccounts/deployment-controller") + if code != 200 or not account.get("metadata", {}).get("uid"): + raise RuntimeError("The actual Kubernetes Deployment controller ServiceAccount is absent") + authorization = {} + for label, group, resource, verb, name in ( + ("createReplicaSetsClusterWide", "apps", "replicasets", "create", None), + ("createPodsClusterWide", "", "pods", "create", None), + ("useRegistrar", "kars.azure.com", "karssreregistrations", "use", "canonical"), + ): + attributes = {"group": group, "resource": resource, "verb": verb} + if name: + attributes["name"] = name + code, body = request(port, "POST", "/apis/authorization.k8s.io/v1/subjectaccessreviews", { + "apiVersion": "authorization.k8s.io/v1", "kind": "SubjectAccessReview", + "spec": {"user": DEPLOYMENT_CONTROLLER, "resourceAttributes": attributes}, + }) + if code != 201 or not isinstance(body.get("status", {}).get("allowed"), bool): + raise RuntimeError("Deployment controller authorization proof failed") + authorization[label] = body["status"]["allowed"] + reports = [] + for private in (False, True): + pod = {"serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]} + if private: + pod["volumes"] = [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}] + obj = {"apiVersion": "apps/v1", "kind": "ReplicaSet", + "metadata": {"name": "e2e-deployment-controller", "namespace": "kars-sre"}, + "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-controller"}}, + "template": {"metadata": {"labels": {"app": "e2e-controller"}}, "spec": pod}}} + code, body = as_tenant(port, "/apis/apps/v1/namespaces/kars-sre/replicasets?dryRun=All", + obj, user=DEPLOYMENT_CONTROLLER) + result = api_result(code, body, policies) + result.update({"case": f"deployment-controller-{'private' if private else 'ordinary'}-replicaset", + "expectedStatus": 201, "matched": code == 201, + "actualControllerAccountExists": True, "authorization": authorization, + "identityMode": "admin-impersonation-of-built-in-controller"}) + reports.append(result) + return reports diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 22cedbfcb..8955917c4 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -233,6 +233,11 @@ def main(root, diagnostics_only, candidate=False, retirement=False): from sre_authority.binding_probe import prove prove(root, port, state, objects, lambda facts: write_report(root, "bootstrap-binding-retirement.json", facts)) + from sre_authority.bootstrap_cases import deployment_controller_cases + controller_cases = deployment_controller_cases(port, policies) + write_report(root, "bootstrap-workload-controller.json", {"cases": controller_cases}) + if not all(case["matched"] for case in controller_cases): + raise RuntimeError("Built-in Deployment controller cannot create the private SRE ReplicaSet") finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 77c4bb7e9..431aa3131 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -132,6 +132,56 @@ def test_arbitrary_failure_is_not_security_proof_in_candidate_cases(self): self.assertFalse(any(case["matched"] for case in cases)) self.assertNotIn("do-not-publish", json.dumps(cases)) + def test_builtin_controller_proof_preserves_real_identity_and_cluster_scope(self): + from sre_authority.bootstrap_cases import DEPLOYMENT_CONTROLLER, deployment_controller_cases + responses = [(200, {"metadata": {"uid": "actual-api-uid"}}), + (201, {"status": {"allowed": True}}), + (201, {"status": {"allowed": False}}), + (201, {"status": {"allowed": False}})] + with patch("sre_authority.bootstrap_cases.request", side_effect=responses) as api, \ + patch("sre_authority.bootstrap_cases.as_tenant", return_value=(201, {"kind": "ReplicaSet"})) as actor: + cases = deployment_controller_cases(1, POLICIES) + self.assertTrue(all(case["matched"] for case in cases)) + self.assertEqual(cases[0]["authorization"], { + "createReplicaSetsClusterWide": True, "createPodsClusterWide": False, "useRegistrar": False}) + for call in api.call_args_list[1:]: + spec = call.args[3]["spec"] + self.assertEqual(spec["user"], DEPLOYMENT_CONTROLLER) + self.assertNotIn("namespace", spec["resourceAttributes"]) + for call in actor.call_args_list: + self.assertEqual(call.kwargs["user"], DEPLOYMENT_CONTROLLER) + self.assertTrue(call.args[1].endswith("?dryRun=All")) + pod = call.args[2]["spec"]["template"]["spec"] + self.assertEqual(pod["schedulerName"], "kars-e2e-admission-never-schedule") + self.assertEqual(pod["containers"][0]["imagePullPolicy"], "Never") + + def test_private_controller_replicaset_denial_is_a_failure_not_a_security_pass(self): + from sre_authority.bootstrap_cases import deployment_controller_cases + policies = {"kars-sre-private-workloads": {}} + for code in (403, 404, 422, 500): + responses = [(200, {"metadata": {"uid": "actual-api-uid"}})] + [ + (201, {"status": {"allowed": allowed}}) for allowed in (True, False, False)] + with patch("sre_authority.bootstrap_cases.request", side_effect=responses), \ + patch("sre_authority.bootstrap_cases.as_tenant", side_effect=[ + (201, {"kind": "ReplicaSet"}), + (code, {"kind": "Status", "reason": "Forbidden", + "message": "kars-sre-private-workloads forbids do-not-publish"})]): + cases = deployment_controller_cases(1, policies) + self.assertTrue(cases[0]["matched"]) + self.assertFalse(cases[1]["matched"]) + self.assertEqual(cases[1]["expectedStatus"], 201) + self.assertNotIn("do-not-publish", json.dumps(cases)) + + def test_controller_proof_requires_actual_account_and_valid_authorization_response(self): + from sre_authority.bootstrap_cases import deployment_controller_cases + for responses in ([(404, {})], [(200, {"metadata": {}})], + [(200, {"metadata": {"uid": "uid"}}), (403, {})], + [(200, {"metadata": {"uid": "uid"}}), (201, {"status": {"allowed": "true"}})]): + with patch("sre_authority.bootstrap_cases.request", side_effect=responses), \ + patch("sre_authority.bootstrap_cases.as_tenant") as actor, self.assertRaises(RuntimeError): + deployment_controller_cases(1, POLICIES) + actor.assert_not_called() + def test_collection_tracks_real_uid_chain_without_logging_other_pods(self): def request(_port, _method, path): if path.endswith("/deployments"): diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 8f9aa0c38..5e26bb9ee 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -373,6 +373,7 @@ def passed(self, message): print(f"SRE-PASS {message}", flush=True) def diagnostics(self): + from .bootstrap_diagnostics import failure_facts self.deadline = max(self.deadline, time.monotonic() + 50) # Status and identities only; never dump Secret bodies or whole Pods. for kind, name, namespace in [("karssreregistrations.kars.azure.com", "canonical", None), @@ -389,7 +390,9 @@ def diagnostics(self): **({"authorityFailure": authority_failure_site(self.root, status.get("detail"))} if kind == "karssreregistrations.kars.azure.com" and status.get("phase") == "Blocked" else {}), "conditions": [{"type": condition.get("type"), "status": condition.get("status"), - "reason": condition.get("reason")} + "reason": condition.get("reason"), + **failure_facts(condition.get("message"), + {f"kars-sre-{name}": {} for name in POLICIES})} for condition in status.get("conditions", [])]}), flush=True) except Exception: print(f"SRE-DIAG {kind}/{name} unavailable", flush=True) From 91e4f32064a5816c6b30cd161a151ba59afba913 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 13:55:21 +0200 Subject: [PATCH 28/62] chore(deps): qualify approved js-yaml 4.3.2 patch locks Pin the three affected consumers to the GHSA-2883-xcg3-v3hh patch. Use temporary hosted lock-only generation against the public registry because the configured local feed exposes proxy URLs and SHA-1 metadata. Require matching public SHA-512 integrity and unchanged versions/tree for every other package before capturing generated locks for review. No audit waiver or policy changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 43 ++++++++++++++++++++++++++++++ mesh-plugin/package.json | 2 +- runtimes/openclaw/package.json | 2 +- tools/headlamp-plugin/package.json | 3 ++- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7157bc56c..5f1c2d518 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -415,6 +415,49 @@ jobs: with: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Use existing Node toolchain for approved js-yaml lock generation + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Generate only approved js-yaml patch lockfiles from the public registry + run: | + for directory in mesh-plugin runtimes/openclaw tools/headlamp-plugin; do + npm install --prefix "$directory" --package-lock-only --ignore-scripts \ + --no-audit --no-fund --registry=https://registry.npmjs.org \ + --cache "$GITHUB_WORKSPACE/.e2e-jsyaml-cache" + done + npm view js-yaml@4.3.2 version dist --json --registry=https://registry.npmjs.org \ + --cache "$GITHUB_WORKSPACE/.e2e-jsyaml-cache" > e2e-jsyaml-dist.json + node <<'NODE' + const fs = require("node:fs"), cp = require("node:child_process"); + const dist = JSON.parse(fs.readFileSync("e2e-jsyaml-dist.json")).dist; + for (const directory of ["mesh-plugin", "runtimes/openclaw", "tools/headlamp-plugin"]) { + const path = `${directory}/package-lock.json`; + const before = JSON.parse(cp.execFileSync("git", ["show", `HEAD:${path}`])); + const after = JSON.parse(fs.readFileSync(path)); + const unchanged = lock => Object.entries(lock.packages) + .filter(([name]) => !name.endsWith("node_modules/js-yaml")) + .map(([name, entry]) => [name, entry.version ?? null]).sort(); + if (JSON.stringify(unchanged(before)) !== JSON.stringify(unchanged(after))) + throw Error("Refusing unrelated package version/tree changes"); + for (const [name, entry] of Object.entries(after.packages)) { + if (name.endsWith("node_modules/js-yaml") && + (entry.version !== "4.3.2" || entry.integrity !== dist.integrity || + entry.resolved !== dist.tarball || !entry.integrity.startsWith("sha512-"))) + throw Error("Generated js-yaml lock does not match public registry metadata"); + } + } + NODE + - name: Capture package-manager generated lockfiles for review + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 + with: + name: sre-jsyaml-locks-${{ github.run_id }} + path: | + mesh-plugin/package-lock.json + runtimes/openclaw/package-lock.json + tools/headlamp-plugin/package-lock.json + e2e-jsyaml-dist.json + retention-days: 7 - name: Check public-schema diagnostic privacy run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test - name: Create the same disposable API server as the real harness diff --git a/mesh-plugin/package.json b/mesh-plugin/package.json index 75b5adb32..08bac40c3 100644 --- a/mesh-plugin/package.json +++ b/mesh-plugin/package.json @@ -48,7 +48,7 @@ "ws": "^8.21.0" }, "overrides": { - "js-yaml": "4.3.1" + "js-yaml": "4.3.2" }, "devDependencies": { "@types/node": "^25.6.0", diff --git a/runtimes/openclaw/package.json b/runtimes/openclaw/package.json index 7096a8d8d..5f7654afc 100644 --- a/runtimes/openclaw/package.json +++ b/runtimes/openclaw/package.json @@ -51,6 +51,6 @@ }, "overrides": { "esbuild": "^0.28.1", - "js-yaml": "4.3.1" + "js-yaml": "4.3.2" } } \ No newline at end of file diff --git a/tools/headlamp-plugin/package.json b/tools/headlamp-plugin/package.json index cc15e06d0..01965b62e 100644 --- a/tools/headlamp-plugin/package.json +++ b/tools/headlamp-plugin/package.json @@ -20,6 +20,7 @@ }, "overrides": { "vitest": "^4.1.8", - "tmp": "^0.2.6" + "tmp": "^0.2.6", + "js-yaml": "4.3.2" } } From c967be0c77576d3883aae95ee5ba238b0bfb4ac3 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 14:03:21 +0200 Subject: [PATCH 29/62] chore(deps): persist verified js-yaml 4.3.2 patch locks Import byte-identical npm-generated public-registry locks from hosted run 34348098105, artifact 10102603454. Independently verify official SHA-512 integrity and unchanged versions/tree for all non-js-yaml packages. Runtime OpenClaw passes 250 tests; mesh passes 68 with three existing skips; both bulk audits report zero blocking advisories. Remove temporary lock-generation CI steps, leaving only the six manifest/lock files changed for the shared patch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 43 ---------------------- mesh-plugin/package-lock.json | 44 +++++++++++------------ runtimes/openclaw/package-lock.json | 47 +++++++++++++------------ tools/headlamp-plugin/package-lock.json | 8 ++--- 4 files changed, 50 insertions(+), 92 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f1c2d518..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -415,49 +415,6 @@ jobs: with: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - - name: Use existing Node toolchain for approved js-yaml lock generation - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "22" - - name: Generate only approved js-yaml patch lockfiles from the public registry - run: | - for directory in mesh-plugin runtimes/openclaw tools/headlamp-plugin; do - npm install --prefix "$directory" --package-lock-only --ignore-scripts \ - --no-audit --no-fund --registry=https://registry.npmjs.org \ - --cache "$GITHUB_WORKSPACE/.e2e-jsyaml-cache" - done - npm view js-yaml@4.3.2 version dist --json --registry=https://registry.npmjs.org \ - --cache "$GITHUB_WORKSPACE/.e2e-jsyaml-cache" > e2e-jsyaml-dist.json - node <<'NODE' - const fs = require("node:fs"), cp = require("node:child_process"); - const dist = JSON.parse(fs.readFileSync("e2e-jsyaml-dist.json")).dist; - for (const directory of ["mesh-plugin", "runtimes/openclaw", "tools/headlamp-plugin"]) { - const path = `${directory}/package-lock.json`; - const before = JSON.parse(cp.execFileSync("git", ["show", `HEAD:${path}`])); - const after = JSON.parse(fs.readFileSync(path)); - const unchanged = lock => Object.entries(lock.packages) - .filter(([name]) => !name.endsWith("node_modules/js-yaml")) - .map(([name, entry]) => [name, entry.version ?? null]).sort(); - if (JSON.stringify(unchanged(before)) !== JSON.stringify(unchanged(after))) - throw Error("Refusing unrelated package version/tree changes"); - for (const [name, entry] of Object.entries(after.packages)) { - if (name.endsWith("node_modules/js-yaml") && - (entry.version !== "4.3.2" || entry.integrity !== dist.integrity || - entry.resolved !== dist.tarball || !entry.integrity.startsWith("sha512-"))) - throw Error("Generated js-yaml lock does not match public registry metadata"); - } - } - NODE - - name: Capture package-manager generated lockfiles for review - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4 - with: - name: sre-jsyaml-locks-${{ github.run_id }} - path: | - mesh-plugin/package-lock.json - runtimes/openclaw/package-lock.json - tools/headlamp-plugin/package-lock.json - e2e-jsyaml-dist.json - retention-days: 7 - name: Check public-schema diagnostic privacy run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test - name: Create the same disposable API server as the real harness diff --git a/mesh-plugin/package-lock.json b/mesh-plugin/package-lock.json index fc0356adc..3aa6dd8c5 100644 --- a/mesh-plugin/package-lock.json +++ b/mesh-plugin/package-lock.json @@ -80,28 +80,6 @@ "node": ">=18.0.0" } }, - "node_modules/@microsoft/agent-governance-sdk/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -834,6 +812,28 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", diff --git a/runtimes/openclaw/package-lock.json b/runtimes/openclaw/package-lock.json index fe71a1a42..831f42762 100644 --- a/runtimes/openclaw/package-lock.json +++ b/runtimes/openclaw/package-lock.json @@ -38,6 +38,7 @@ "devDependencies": { "@types/node": "^25.6.0", "@types/ws": "^8", + "oxlint": "^0.16.0", "typescript": "^5.7", "vitest": "^4.1.8" }, @@ -549,29 +550,6 @@ "node": ">=18.0.0" } }, - "node_modules/@microsoft/agent-governance-sdk/node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", @@ -1350,6 +1328,29 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", diff --git a/tools/headlamp-plugin/package-lock.json b/tools/headlamp-plugin/package-lock.json index f4e0e5616..442fcb529 100644 --- a/tools/headlamp-plugin/package-lock.json +++ b/tools/headlamp-plugin/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "devDependencies": { "@kinvolk/headlamp-plugin": "^0.13.0", - "vite": "^6.4.3" + "vite": "6.4.3" } }, "node_modules/@adobe/css-tools": { @@ -11457,9 +11457,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, "funding": [ { From c08465a5f7e3957b3594c5b37088c56d00290449 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 14:17:36 +0200 Subject: [PATCH 30/62] fix(sre): authorize the Deployment controller's ReplicaSet handoff Actual Kubernetes 1.31 evidence shows the built-in Deployment controller can create ReplicaSets cluster-wide but cannot create Pods or use SRE registrar authority. Permit that existing cluster-wide capability only for apps/replicasets admission, retaining all other predicates and granting no RBAC privileges. Add real UID-linked private Deployment-to-ReplicaSet-to-unscheduled-Pod acceptance, alongside unchanged tenant denials. Python regressions: 67 passed; Helm lint and exact rendered-policy preservation checks passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../templates/sre-authority-consumers.yaml | 4 ++ docs/how-to/sre-authority.md | 4 ++ tests/e2e/sre_authority/bootstrap_cases.py | 70 ++++++++++++++++++- tests/e2e/sre_authority/bootstrap_probe.py | 4 +- .../e2e/sre_authority/bootstrap_probe_test.py | 67 ++++++++++++++++++ 5 files changed, 147 insertions(+), 2 deletions(-) diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml index a53da422f..6ae135bfc 100644 --- a/deploy/helm/kars/templates/sre-authority-consumers.yaml +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -173,6 +173,10 @@ spec: !variables.privateMaterial || authorizer.group('kars.azure.com').resource('karssreregistrations').name('canonical').check('use').allowed() || authorizer.group('').resource('pods').check('create').allowed() + {{- if eq $kind "workloads" }} + || (request.resource.group == 'apps' && request.resource.resource == 'replicasets' && + authorizer.group('apps').resource('replicasets').check('create').allowed()) + {{- end }} message: "Private SRE workload templates require registrar or cluster-wide workload-controller authority" reason: Forbidden --- diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index d26426f89..dcaa6da4c 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -209,6 +209,10 @@ CronJob templates from laundering a private mount through Kubernetes workload controllers. Exec/attach/port-forward into the private SRE runtime requires registrar authority. Cluster-wide workload controllers remain trusted; installing a custom privileged controller is a cluster-operator action. +The Deployment-controller handoff is authorized only for ReplicaSet requests +and requires cluster-wide `apps/replicasets` CREATE authority; namespaced +workload permissions are insufficient. It does not grant the Deployment +controller Pod CREATE or registrar authority. The proxy checks current registration and live UID/claim authority. It permits the bounded first-party diagnostic read/log/metrics paths and Pending-only diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index 04422340a..9db6cc47f 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -5,10 +5,11 @@ import copy import json +import time from urllib.error import HTTPError from urllib.request import Request, build_opener, ProxyHandler -from .bootstrap_diagnostics import api_result +from .bootstrap_diagnostics import api_result, object_status from .bootstrap_probe import upsert from .registration_schema import request @@ -139,3 +140,70 @@ def deployment_controller_cases(port, policies): "identityMode": "admin-impersonation-of-built-in-controller"}) reports.append(result) return reports + + +def private_controller_chain(port, policies, report): + namespace, name = "kars-sre", "e2e-private-controller-chain" + selector = {"app": name} + deployment = {"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": name, "namespace": namespace}, + "spec": {"replicas": 1, "selector": {"matchLabels": selector}, + "template": {"metadata": {"labels": selector}, "spec": { + "serviceAccountName": "sandbox", "automountServiceAccountToken": False, + "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never", + "volumeMounts": [{"name": "private", "mountPath": "/private", "readOnly": True}]}], + "volumes": [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}], + }}}} + path = f"/apis/apps/v1/namespaces/{namespace}/deployments" + code, created = request(port, "POST", path, deployment) + if code != 201 or not created.get("metadata", {}).get("uid"): + report({"deploymentCreate": api_result(code, created, policies)}) + raise RuntimeError("Registrar-authorized private Deployment CREATE failed") + uid = created["metadata"]["uid"] + deadline = time.monotonic() + 45 + snapshot = {} + while time.monotonic() < deadline: + code, current = request(port, "GET", f"{path}/{name}") + if code != 200 or current.get("metadata", {}).get("uid") != uid: + raise RuntimeError("Private controller-chain Deployment disappeared or was replaced") + code, replicasets = request(port, "GET", + f"/apis/apps/v1/namespaces/{namespace}/replicasets?labelSelector=app%3D{name}") + if code != 200 or not isinstance(replicasets.get("items"), list): + raise RuntimeError("Private controller-chain ReplicaSet inspection failed") + owned = [obj for obj in replicasets["items"] if any( + owner.get("uid") == uid and owner.get("controller") is True + for owner in obj.get("metadata", {}).get("ownerReferences", []))] + owners = {obj["metadata"]["uid"] for obj in owned} + code, pods = request(port, "GET", f"/api/v1/namespaces/{namespace}/pods?labelSelector=app%3D{name}") + if code != 200 or not isinstance(pods.get("items"), list): + raise RuntimeError("Private controller-chain Pod inspection failed") + children = [obj for obj in pods["items"] if any( + owner.get("uid") in owners and owner.get("controller") is True + for owner in obj.get("metadata", {}).get("ownerReferences", []))] + snapshot = {"deployment": object_status(current, policies), + "replicaSets": [object_status(dict(obj, kind="ReplicaSet"), policies) for obj in owned], + "pods": [object_status(dict(obj, kind="Pod"), policies) for obj in children], + "privateMountPreserved": False, "noWorkloadExecution": False} + if children: + snapshot["privateMountPreserved"] = all( + obj["spec"].get("volumes") and any( + volume.get("secret", {}).get("secretName") == "sre-api-router-identity" + for volume in obj["spec"]["volumes"]) + and any(mount.get("name") == "private" and mount.get("mountPath") == "/private" + and mount.get("readOnly") is True + for container in obj["spec"].get("containers", []) + for mount in container.get("volumeMounts", [])) + for obj in children) + snapshot["noWorkloadExecution"] = all( + obj["spec"].get("schedulerName") == "kars-e2e-admission-never-schedule" + and not obj["spec"].get("nodeName") and not obj.get("status", {}).get("containerStatuses") + for obj in children) + report(snapshot) + if not snapshot["privateMountPreserved"] or not snapshot["noWorkloadExecution"]: + raise RuntimeError("Private controller-chain proof changed its protected template or executed a workload") + return + time.sleep(0.5) + report(snapshot) + raise RuntimeError("Actual private Deployment/ReplicaSet controllers did not create the admission-only Pod") diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 8955917c4..79abbcf82 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -233,11 +233,13 @@ def main(root, diagnostics_only, candidate=False, retirement=False): from sre_authority.binding_probe import prove prove(root, port, state, objects, lambda facts: write_report(root, "bootstrap-binding-retirement.json", facts)) - from sre_authority.bootstrap_cases import deployment_controller_cases + from sre_authority.bootstrap_cases import deployment_controller_cases, private_controller_chain controller_cases = deployment_controller_cases(port, policies) write_report(root, "bootstrap-workload-controller.json", {"cases": controller_cases}) if not all(case["matched"] for case in controller_cases): raise RuntimeError("Built-in Deployment controller cannot create the private SRE ReplicaSet") + private_controller_chain(port, policies, + lambda facts: write_report(root, "bootstrap-private-controller-chain.json", facts)) finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 431aa3131..e6b9eb2b7 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -182,6 +182,73 @@ def test_controller_proof_requires_actual_account_and_valid_authorization_respon deployment_controller_cases(1, POLICIES) actor.assert_not_called() + def private_chain_api(self, *, pod_change=None, replaced=False, no_child=False): + created = {} + def api(_port, method, path, obj=None): + if method == "POST": + created.update(copy.deepcopy(obj)) + created["metadata"]["uid"] = "deployment-uid" + return 201, created + if "/deployments/" in path: + current = copy.deepcopy(created) + if replaced: + current["metadata"]["uid"] = "replacement" + current["status"] = {"conditions": [{"type": "Progressing", "status": "False", + "reason": "ReplicaSetCreateError", "message": "kars-sre-private-workloads forbidden do-not-publish"}]} + return 200, current + if "/replicasets?" in path: + return 200, {"items": [{"metadata": {"name": "owned-rs", "uid": "replicaset-uid", + "ownerReferences": [{"uid": "deployment-uid", "controller": True}]}}]} + if "/pods?" in path: + pod = {"metadata": {"name": "owned-pod", "uid": "pod-uid", + "ownerReferences": [{"uid": "replicaset-uid", "controller": True}]}, + "spec": copy.deepcopy(created["spec"]["template"]["spec"])} + if pod_change: + pod_change(pod) + return 200, {"items": [] if no_child else [pod, {"metadata": {"name": "do-not-publish", + "uid": "foreign", "ownerReferences": [{"uid": "not-ours", "controller": True}]}}]} + raise AssertionError("Unexpected private chain API request") + return api + + def test_actual_private_chain_requires_deployment_replicaset_pod_uid_ownership(self): + from sre_authority.bootstrap_cases import private_controller_chain + reports = [] + with patch("sre_authority.bootstrap_cases.request", side_effect=self.private_chain_api()) as api: + private_controller_chain(1, {"kars-sre-private-workloads": {}}, reports.append) + self.assertTrue(reports[0]["privateMountPreserved"]) + self.assertTrue(reports[0]["noWorkloadExecution"]) + self.assertEqual([pod["uid"] for pod in reports[0]["pods"]], ["pod-uid"]) + self.assertNotIn("do-not-publish", json.dumps(reports)) + self.assertEqual([call.args[1] for call in api.call_args_list], ["POST", "GET", "GET", "GET"]) + self.assertTrue(api.call_args_list[0].args[2].endswith("/deployments")) + + def test_private_chain_rejects_replacement_missing_mount_or_execution(self): + from sre_authority.bootstrap_cases import private_controller_chain + variants = [ + {"replaced": True}, + {"pod_change": lambda pod: pod["spec"].update(nodeName="scheduled-node")}, + {"pod_change": lambda pod: pod["spec"].pop("volumes")}, + {"pod_change": lambda pod: pod["spec"]["containers"][0].pop("volumeMounts")}, + ] + for variant in variants: + with patch("sre_authority.bootstrap_cases.request", side_effect=self.private_chain_api(**variant)), \ + self.subTest(variant=variant), self.assertRaises(RuntimeError): + private_controller_chain(1, POLICIES, lambda _report: None) + + def test_private_chain_timeout_reports_sanitized_blocker_not_success(self): + from sre_authority.bootstrap_cases import private_controller_chain + for variant in ({"no_child": True}, + {"pod_change": lambda pod: pod["metadata"]["ownerReferences"][0].update(uid="foreign")}, + {"pod_change": lambda pod: pod["metadata"]["ownerReferences"][0].update(controller=False)}): + reports = [] + with patch("sre_authority.bootstrap_cases.request", side_effect=self.private_chain_api(**variant)), \ + patch("sre_authority.bootstrap_cases.time.monotonic", side_effect=[0, 1, 46]), \ + patch("sre_authority.bootstrap_cases.time.sleep"), self.assertRaises(RuntimeError): + private_controller_chain(1, {"kars-sre-private-workloads": {}}, reports.append) + self.assertFalse(reports[0]["privateMountPreserved"]) + self.assertFalse(reports[0]["noWorkloadExecution"]) + self.assertNotIn("do-not-publish", json.dumps(reports)) + def test_collection_tracks_real_uid_chain_without_logging_other_pods(self): def request(_port, _method, path): if path.endswith("/deployments"): From 342179136fdc5101e7771f63a445448f372d4132 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 15:04:24 +0200 Subject: [PATCH 31/62] test(e2e): isolate distroless SRE readiness command failures Full Kind now creates the private SRE Pod but the running router remains NotReady. Add allowlisted bare/absolute command and verified loopback-TLS diagnostics. A temporary hosted smoke probe reuses SHA256-verified binaries from run 34350195081 only after proving unchanged production sources, without Cargo or private credentials. Preserve the actual readiness gate and all production code. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 43 +++++++++++++++++++ .../sre_authority/bootstrap_diagnostics.py | 11 +++++ .../e2e/sre_authority/bootstrap_probe_test.py | 13 +++++- tests/e2e/sre_authority/common.py | 24 +++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7157bc56c..2c6f6ee62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -400,6 +400,9 @@ jobs: name: SRE CRD API Schema runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read + actions: read env: KUBECONFIG: ${{ github.workspace }}/.e2e-sre-schema-kubeconfig PYTHONDONTWRITEBYTECODE: "1" @@ -415,6 +418,45 @@ jobs: with: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Verify unchanged production sources for existing CI router artifact + run: | + git fetch --no-tags --depth=1 origin a2b394a6ad77acc72a197255f25438fed40d41da + changed=$(git diff --name-only a2b394a6ad77acc72a197255f25438fed40d41da HEAD) + if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|\.github/workflows/ci\.yml$|$)'; then + echo "Refusing to diagnose with a binary from different production sources" + exit 1 + fi + - name: Read verified existing CI binaries for command-only readiness diagnosis + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: kars-binaries-a2b394a6ad77acc72a197255f25438fed40d41da + run-id: 34350195081 + github-token: ${{ github.token }} + path: .e2e-router-probe/bin + - name: Diagnose bare versus absolute readiness command in the unchanged distroless image + run: | + (cd .e2e-router-probe/bin && sha256sum --check SHA256SUMS) + chmod +x .e2e-router-probe/bin/amd64/kars-inference-router + docker build --quiet --build-arg TARGETARCH=amd64 \ + --build-arg BIN_PATH_PREFIX=.e2e-router-probe/bin \ + -f inference-router/Dockerfile -t kars-inference-router:e2e . + bare=0 + docker run --rm --entrypoint kars-inference-router kars-inference-router:e2e sre-ready \ + > .e2e-router-probe/bare.log 2>&1 || bare=$? + absolute=0 + docker run --rm --entrypoint /usr/local/bin/kars-inference-router kars-inference-router:e2e sre-ready \ + > .e2e-router-probe/absolute.log 2>&1 || absolute=$? + PYTHONPATH=tests/e2e python3 - "$bare" "$absolute" <<'PY' + import sys + from pathlib import Path + from sre_authority.bootstrap_diagnostics import probe_command_result + from sre_authority.registration_schema import write_report + facts = {name: probe_command_result(int(code), Path(f".e2e-router-probe/{name}.log").read_text()) + for name, code in zip(("bare", "absolute"), sys.argv[1:])} + write_report(Path.cwd(), "router-readiness-command.json", facts) + assert facts["absolute"]["category"] == "probe-not-ready" + assert facts["bare"]["category"] in ("probe-not-ready", "executable-not-found") + PY - name: Check public-schema diagnostic privacy run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test - name: Create the same disposable API server as the real harness @@ -445,6 +487,7 @@ jobs: path: | e2e-sre-schema-diag/versions.json e2e-sre-schema-diag/legacy-helm-readiness.json + e2e-sre-schema-diag/router-readiness-command.json e2e-sre-schema-diag/validation.json e2e-sre-schema-diag/validation-instances.json e2e-sre-schema-diag/namespace-accessor-candidate.json diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index e4bb2c6ac..debe823ba 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -21,6 +21,17 @@ } +def probe_command_result(code, output): + category = "unclassified" + if code == 0: + category = "succeeded" + elif "executable file not found" in output: + category = "executable-not-found" + elif code == 1 and not output.strip(): + category = "probe-not-ready" + return {"exitCode": code, "category": category} + + def identifier(value): return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index e6b9eb2b7..9ceedf28e 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import patch -from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, public_stack_facts +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, probe_command_result, public_stack_facts from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ @@ -15,6 +15,17 @@ class BootstrapProofTests(unittest.TestCase): + def test_probe_diagnostics_never_publish_executable_output(self): + for code, output, category in ( + (127, "exec: executable file not found in $PATH do-not-publish", "executable-not-found"), + (1, "", "probe-not-ready"), + (0, "do-not-publish", "succeeded"), + (1, "do-not-publish", "unclassified"), + ): + facts = probe_command_result(code, output) + self.assertEqual(facts, {"exitCode": code, "category": category}) + self.assertNotIn("do-not-publish", json.dumps(facts)) + def test_failure_metadata_keeps_public_cause_not_body_or_credentials(self): message = ('Error creating Pod: kars-sre-private-mounts evaluation failed: no such key: namespace; ' 'token=do-not-publish argv=do-not-publish') diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 5e26bb9ee..1745752d2 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -407,9 +407,33 @@ def diagnostics(self): "state": {kind: {key: value.get(key) for key in ("reason", "exitCode") if key in value} for kind, value in container.get("state", {}).items()}} for container in containers]}), flush=True) + router = next((container for container in status.get("containerStatuses", []) + if container["name"] == "inference-router"), None) + if router and not router.get("ready") and "running" in router.get("state", {}): + self.readiness_diagnostics(pod) except Exception: print("SRE-DIAG runtime Pod status unavailable", flush=True) + def readiness_diagnostics(self, pod): + from .bootstrap_diagnostics import probe_command_result + facts = {"kind": "RouterReadiness", "podUid": pod["metadata"]["uid"]} + try: + for label, executable in (("configuredCommand", "kars-inference-router"), + ("absoluteCommand", "/usr/local/bin/kars-inference-router")): + result = self.k("exec", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", + "--", executable, "sre-ready", expected=None, timeout=10) + facts[label] = probe_command_result(result.returncode, result.stdout + result.stderr) + ca = self.k("exec", "-n", RUNTIME, pod["metadata"]["name"], "-c", "agent", + "--", "cat", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", timeout=10) + context = ssl.create_default_context(cadata=ca) + with self.port_forward(pod["metadata"]["name"]) as port, \ + self.httpx.Client(verify=context, timeout=5, trust_env=False) as client: + response = client.get(f"https://127.0.0.1:{port}/readyz") + facts["verifiedLoopbackTlsStatus"] = response.status_code + except Exception as error: + facts["diagnosticError"] = type(error).__name__ + print("SRE-DIAG", json.dumps(facts), flush=True) + def close(self): for process in self.processes: process.terminate() From f319d8352fac16e0709a7f5c614f2da7fb566dce Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 15:48:17 +0200 Subject: [PATCH 32/62] fix(sre): expose bounded authority-readiness failure diagnostics Verified-image and real Pod evidence rule out executable lookup; certificate-verified loopback readiness requests time out while registration remains Ready. Log only fixed authority stages/categories, HTTP status, timeout/connect booleans and slow-check duration. Never log URLs, response bodies, errors or credentials; retain every authorization decision and timeout. Add log-boundary regressions and remove the completed temporary image probe. Python: 69 passed; changed Rust files formatted, with all compilation/tests delegated to existing hosted CI and no local Cargo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 43 ----------- inference-router/src/sre_proxy/backend.rs | 72 ++++++++++++++----- inference-router/src/sre_proxy/mod.rs | 49 +++++++++++-- inference-router/src/sre_proxy/tests.rs | 20 ++++++ .../sre_authority/bootstrap_diagnostics.py | 34 +++++++++ .../e2e/sre_authority/bootstrap_probe_test.py | 20 +++++- tests/e2e/sre_authority/common.py | 8 ++- 7 files changed, 179 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c6f6ee62..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -400,9 +400,6 @@ jobs: name: SRE CRD API Schema runs-on: ubuntu-latest timeout-minutes: 10 - permissions: - contents: read - actions: read env: KUBECONFIG: ${{ github.workspace }}/.e2e-sre-schema-kubeconfig PYTHONDONTWRITEBYTECODE: "1" @@ -418,45 +415,6 @@ jobs: with: version: v1.30.5 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - - name: Verify unchanged production sources for existing CI router artifact - run: | - git fetch --no-tags --depth=1 origin a2b394a6ad77acc72a197255f25438fed40d41da - changed=$(git diff --name-only a2b394a6ad77acc72a197255f25438fed40d41da HEAD) - if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|\.github/workflows/ci\.yml$|$)'; then - echo "Refusing to diagnose with a binary from different production sources" - exit 1 - fi - - name: Read verified existing CI binaries for command-only readiness diagnosis - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: kars-binaries-a2b394a6ad77acc72a197255f25438fed40d41da - run-id: 34350195081 - github-token: ${{ github.token }} - path: .e2e-router-probe/bin - - name: Diagnose bare versus absolute readiness command in the unchanged distroless image - run: | - (cd .e2e-router-probe/bin && sha256sum --check SHA256SUMS) - chmod +x .e2e-router-probe/bin/amd64/kars-inference-router - docker build --quiet --build-arg TARGETARCH=amd64 \ - --build-arg BIN_PATH_PREFIX=.e2e-router-probe/bin \ - -f inference-router/Dockerfile -t kars-inference-router:e2e . - bare=0 - docker run --rm --entrypoint kars-inference-router kars-inference-router:e2e sre-ready \ - > .e2e-router-probe/bare.log 2>&1 || bare=$? - absolute=0 - docker run --rm --entrypoint /usr/local/bin/kars-inference-router kars-inference-router:e2e sre-ready \ - > .e2e-router-probe/absolute.log 2>&1 || absolute=$? - PYTHONPATH=tests/e2e python3 - "$bare" "$absolute" <<'PY' - import sys - from pathlib import Path - from sre_authority.bootstrap_diagnostics import probe_command_result - from sre_authority.registration_schema import write_report - facts = {name: probe_command_result(int(code), Path(f".e2e-router-probe/{name}.log").read_text()) - for name, code in zip(("bare", "absolute"), sys.argv[1:])} - write_report(Path.cwd(), "router-readiness-command.json", facts) - assert facts["absolute"]["category"] == "probe-not-ready" - assert facts["bare"]["category"] in ("probe-not-ready", "executable-not-found") - PY - name: Check public-schema diagnostic privacy run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test - name: Create the same disposable API server as the real harness @@ -487,7 +445,6 @@ jobs: path: | e2e-sre-schema-diag/versions.json e2e-sre-schema-diag/legacy-helm-readiness.json - e2e-sre-schema-diag/router-readiness-command.json e2e-sre-schema-diag/validation.json e2e-sre-schema-diag/validation-instances.json e2e-sre-schema-diag/namespace-accessor-candidate.json diff --git a/inference-router/src/sre_proxy/backend.rs b/inference-router/src/sre_proxy/backend.rs index 0cce84cf2..41c582543 100644 --- a/inference-router/src/sre_proxy/backend.rs +++ b/inference-router/src/sre_proxy/backend.rs @@ -37,6 +37,23 @@ struct Token { expiry: DateTime, } +fn transport_failure(stage: &'static str, error: &reqwest::Error) { + tracing::warn!( + stage, + timed_out = error.is_timeout(), + connect_error = error.is_connect(), + "SRE authority transport failure" + ); +} + +fn denied_response(stage: &'static str, status: reqwest::StatusCode) { + tracing::warn!( + stage, + http_status = status.as_u16(), + "SRE authority request denied" + ); +} + pub(super) struct Backend { pub config: Config, client: reqwest::Client, @@ -180,7 +197,7 @@ impl Backend { Ok(token.value.clone()) } - async fn metadata_json(&self, path: &str) -> Result { + async fn metadata_json(&self, path: &str, stage: &'static str) -> Result { let response = self .client .get(format!( @@ -192,8 +209,12 @@ impl Backend { .header("accept", "application/json") .send() .await - .map_err(|_| "SRE authority read failed")?; + .map_err(|error| { + transport_failure(stage, &error); + "SRE authority read failed" + })?; if !response.status().is_success() { + denied_response(stage, response.status()); return Err("SRE authority read denied".into()); } response @@ -204,7 +225,10 @@ impl Backend { pub(super) async fn authorize(&self) -> Result<(), String> { let reg = self - .metadata_json("/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical") + .metadata_json( + "/apis/kars.azure.com/v1alpha1/karssreregistrations/canonical", + "registration", + ) .await?; if reg["metadata"]["uid"] != self.config.registration_uid || !reg["metadata"]["deletionTimestamp"].is_null() @@ -221,10 +245,10 @@ impl Backend { return Err("SRE authority is no longer current".into()); } let namespace = self - .metadata_json(&format!( - "/api/v1/namespaces/{}", - self.config.runtime_namespace - )) + .metadata_json( + &format!("/api/v1/namespaces/{}", self.config.runtime_namespace), + "namespace", + ) .await?; let annotations = &namespace["metadata"]["annotations"]; if namespace["metadata"]["uid"] != self.config.namespace_uid @@ -241,10 +265,13 @@ impl Backend { return Err("SRE runtime namespace ownership changed".into()); } let sandbox = self - .metadata_json(&format!( - "/apis/kars.azure.com/v1alpha1/namespaces/{}/karssandboxes/{}", - self.config.source.namespace, self.config.source.name - )) + .metadata_json( + &format!( + "/apis/kars.azure.com/v1alpha1/namespaces/{}/karssandboxes/{}", + self.config.source.namespace, self.config.source.name + ), + "source", + ) .await?; if sandbox["metadata"]["uid"] != self.config.source.uid || !sandbox["metadata"]["deletionTimestamp"].is_null() @@ -254,10 +281,13 @@ impl Backend { return Err("SRE source identity changed".into()); } let sa = self - .metadata_json(&format!( - "/api/v1/namespaces/{}/serviceaccounts/sre-api-router", - self.config.runtime_namespace - )) + .metadata_json( + &format!( + "/api/v1/namespaces/{}/serviceaccounts/sre-api-router", + self.config.runtime_namespace + ), + "service-account", + ) .await?; if sa["metadata"]["uid"] != self.config.service_account_uid || !sa["metadata"]["deletionTimestamp"].is_null() @@ -280,8 +310,12 @@ impl Backend { .json(&review) .send() .await - .map_err(|_| "SRE privacy authorization transport failure")?; + .map_err(|error| { + transport_failure("privacy-review", &error); + "SRE privacy authorization transport failure" + })?; if !response.status().is_success() { + denied_response("privacy-review", response.status()); return Err("SRE privacy authorization review denied".into()); } let response: Value = response @@ -304,8 +338,12 @@ impl Backend { ) .send() .await - .map_err(|_| "SRE credential metadata inventory failed")?; + .map_err(|error| { + transport_failure("credential-metadata", &error); + "SRE credential metadata inventory failed" + })?; if !response.status().is_success() { + denied_response("credential-metadata", response.status()); return Err("SRE credential metadata inventory denied".into()); } let metadata: Value = response diff --git a/inference-router/src/sre_proxy/mod.rs b/inference-router/src/sre_proxy/mod.rs index c539361a6..55160ad82 100644 --- a/inference-router/src/sre_proxy/mod.rs +++ b/inference-router/src/sre_proxy/mod.rs @@ -53,6 +53,29 @@ fn error(status: StatusCode, message: &str) -> Response { .into_response() } +fn readiness_failure_category(reason: &str) -> &'static str { + match reason { + "SRE authority read failed" => "authority-transport", + "SRE authority read denied" => "authority-denied", + "SRE authority is no longer current" => "registration-stale", + "SRE runtime namespace ownership changed" => "namespace-claim", + "SRE source identity changed" => "source-identity", + "Private SRE ServiceAccount was replaced" => "service-account", + "SRE privacy authorization transport failure" => "privacy-transport", + "SRE privacy authorization review denied" => "privacy-denied", + "Legacy SRE Secret get/list/watch authorization is allowed or indeterminate" => { + "privacy-not-denied" + } + "SRE credential metadata inventory failed" => "metadata-transport", + "SRE credential metadata inventory denied" => "metadata-denied", + "Unsafe legacy SRE token Secret alias exists; operator review required, no Secret adopted or deleted" => { + "legacy-alias" + } + "Private SRE Kubernetes credential expired; no ambient fallback" => "credential-expired", + _ => "unclassified", + } +} + async fn ready(State(proxy): State) -> Response { let Ok(_permit) = proxy.capacity.try_acquire() else { return error( @@ -60,16 +83,32 @@ async fn ready(State(proxy): State) -> Response { "SRE proxy capacity is exhausted", ); }; - match proxy.backend.authorize().await { + let started = std::time::Instant::now(); + let authorization = proxy.backend.authorize().await; + let elapsed_seconds = started.elapsed().as_secs(); + if elapsed_seconds >= 2 { + tracing::warn!( + elapsed_seconds, + authorized = authorization.is_ok(), + "SRE readiness authority slow" + ); + } + match authorization { Ok(()) => ( StatusCode::OK, axum::Json(serde_json::json!({"ready":true})), ) .into_response(), - Err(_) => error( - StatusCode::SERVICE_UNAVAILABLE, - "SRE authority is not ready", - ), + Err(reason) => { + tracing::warn!( + category = readiness_failure_category(&reason), + "SRE readiness authority rejected" + ); + error( + StatusCode::SERVICE_UNAVAILABLE, + "SRE authority is not ready", + ) + } } } diff --git a/inference-router/src/sre_proxy/tests.rs b/inference-router/src/sre_proxy/tests.rs index a9ab83633..5ebd30586 100644 --- a/inference-router/src/sre_proxy/tests.rs +++ b/inference-router/src/sre_proxy/tests.rs @@ -12,6 +12,26 @@ mod request_boundary; const PRIVATE_VALUE: &str = "PRIVATE_OPERATOR_CONTROL_VALUE"; +#[test] +fn readiness_rejection_categories_exclude_error_details() { + assert_eq!( + readiness_failure_category("SRE authority read failed"), + "authority-transport" + ); + assert_eq!( + readiness_failure_category("SRE authority read denied"), + "authority-denied" + ); + for message in [ + PRIVATE_VALUE.to_string(), + format!("SRE authority read failed: {PRIVATE_VALUE}"), + format!("{PRIVATE_VALUE} SRE authority read denied"), + ] { + assert_eq!(readiness_failure_category(&message), "unclassified"); + assert!(!readiness_failure_category(&message).contains(PRIVATE_VALUE)); + } +} + struct Fixture { _upstream: MockServer, directory: tempfile::TempDir, diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index debe823ba..1eef169f5 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -32,6 +32,40 @@ def probe_command_result(code, output): return {"exitCode": code, "category": category} +def router_readiness_facts(text): + facts = [] + stages = ("registration", "namespace", "source", "service-account", "privacy-review", "credential-metadata") + categories = ("authority-transport", "authority-denied", "registration-stale", "namespace-claim", + "source-identity", "service-account", "privacy-transport", "privacy-denied", + "privacy-not-denied", "metadata-transport", "metadata-denied", "legacy-alias", + "credential-expired", "unclassified") + for line in text.splitlines()[-150:]: + line = re.sub(r"\x1b\[[0-9;]*m", "", line) + if not any(message in line for message in ("SRE authority transport failure", + "SRE authority request denied", + "SRE readiness authority rejected", + "SRE readiness authority slow")): + continue + value = {} + for key, allowed in (("stage", stages), ("category", categories)): + match = re.search(rf'\b{key}="?({"|".join(allowed)})"?(?:\s|$)', line) + if match: + value[key] = match[1] + for key in ("timed_out", "connect_error", "authorized"): + match = re.search(rf"\b{key}=(true|false)(?:\s|$)", line) + if match: + value[key] = match[1] == "true" + status = re.search(r"\bhttp_status=([1-5][0-9]{2})(?:\s|$)", line) + if status: + value["httpStatus"] = int(status[1]) + elapsed = re.search(r"\belapsed_seconds=([0-9]{1,3})(?:\s|$)", line) + if elapsed: + value["elapsedSeconds"] = int(elapsed[1]) + if value and value not in facts: + facts.append(value) + return facts[:16] + + def identifier(value): return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 9ceedf28e..9cdbb7dba 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import patch -from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, probe_command_result, public_stack_facts +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, probe_command_result, public_stack_facts, router_readiness_facts from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ @@ -15,6 +15,24 @@ class BootstrapProofTests(unittest.TestCase): + def test_router_readiness_logs_expose_only_fixed_stage_status_and_timeout_facts(self): + text = ( + '\x1b[33mSRE authority transport failure\x1b[0m stage="registration" timed_out=true connect_error=false token=do-not-publish\n' + 'SRE authority request denied stage="privacy-review" http_status=403 body=do-not-publish\n' + 'SRE readiness authority rejected category="authority-transport" do-not-publish\n' + 'SRE readiness authority slow elapsed_seconds=20 authorized=false do-not-publish\n' + 'unrelated log stage="source" do-not-publish\n' + 'SRE authority request denied stage="do-not-publish" http_status=999\n' + ) + facts = router_readiness_facts(text) + self.assertEqual(facts, [ + {"stage": "registration", "timed_out": True, "connect_error": False}, + {"stage": "privacy-review", "httpStatus": 403}, + {"category": "authority-transport"}, + {"authorized": False, "elapsedSeconds": 20}, + ]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + def test_probe_diagnostics_never_publish_executable_output(self): for code, output, category in ( (127, "exec: executable file not found in $PATH do-not-publish", "executable-not-found"), diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 1745752d2..f58094450 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -415,7 +415,7 @@ def diagnostics(self): print("SRE-DIAG runtime Pod status unavailable", flush=True) def readiness_diagnostics(self, pod): - from .bootstrap_diagnostics import probe_command_result + from .bootstrap_diagnostics import probe_command_result, router_readiness_facts facts = {"kind": "RouterReadiness", "podUid": pod["metadata"]["uid"]} try: for label, executable in (("configuredCommand", "kars-inference-router"), @@ -432,6 +432,12 @@ def readiness_diagnostics(self, pod): facts["verifiedLoopbackTlsStatus"] = response.status_code except Exception as error: facts["diagnosticError"] = type(error).__name__ + try: + logs = self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", + "--tail=150", timeout=10) + facts["authorityChecks"] = router_readiness_facts(logs) + except Exception: + facts["authorityChecksUnavailable"] = True print("SRE-DIAG", json.dumps(facts), flush=True) def close(self): From 7c7aefb547b98ce334fc7afba4baa287bdaa644c Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 16:34:15 +0200 Subject: [PATCH 33/62] test(e2e): parse actual JSON readiness diagnostics The router's production tracing format is JSON; accept only its exact diagnostic messages and allowlisted typed fields, never other log fields. Temporarily start the full Kind replay with SHA256-verified binaries from the successful f319d835 Rust gate after exact production-tree equivalence, while all ordinary Rust checks still run independently. Restore normal build dependency before final qualification. Python: 70 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 16 ++++++++-- .../sre_authority/bootstrap_diagnostics.py | 30 ++++++++++++++++--- .../e2e/sre_authority/bootstrap_probe_test.py | 18 +++++++++++ 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7157bc56c..882e4b14a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -602,7 +602,7 @@ jobs: e2e-kind: name: E2E (Kind) - needs: [changes, build-rust] + needs: [changes] # Phase 3 S4: closes the audit gap "make test-e2e is not in CI". # Runs on every push to dev/main, manual dispatch, and PRs that # touch the runtime surface area (controller, router, helm chart, @@ -702,8 +702,20 @@ jobs: if: steps.paths.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kars-binaries-${{ github.sha }} + name: kars-binaries-921a38354ff2cdcb363f15974a9d4bfd449641bc + run-id: 34359455786 + github-token: ${{ github.token }} path: ./bin/ + - name: Verify diagnostic binary production equivalence and checksums + if: steps.paths.outputs.run == 'true' + run: | + git fetch --no-tags --depth=1 origin 921a38354ff2cdcb363f15974a9d4bfd449641bc + changed=$(git diff --name-only 921a38354ff2cdcb363f15974a9d4bfd449641bc HEAD) + if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|\.github/workflows/ci\.yml$|$)'; then + echo "Refusing diagnostic binaries from different production sources" + exit 1 + fi + (cd bin && sha256sum --check SHA256SUMS) - name: chmod binaries if: steps.paths.outputs.run == 'true' # Per-arch layout (./bin/amd64/) matches release-internal.yml's diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 1eef169f5..5f610dca6 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -3,6 +3,7 @@ """Allowlisted public admission evidence; never dump Pod specs, argv or bodies.""" +import json import re import subprocess @@ -39,12 +40,33 @@ def router_readiness_facts(text): "source-identity", "service-account", "privacy-transport", "privacy-denied", "privacy-not-denied", "metadata-transport", "metadata-denied", "legacy-alias", "credential-expired", "unclassified") + messages = ("SRE authority transport failure", "SRE authority request denied", + "SRE readiness authority rejected", "SRE readiness authority slow") for line in text.splitlines()[-150:]: line = re.sub(r"\x1b\[[0-9;]*m", "", line) - if not any(message in line for message in ("SRE authority transport failure", - "SRE authority request denied", - "SRE readiness authority rejected", - "SRE readiness authority slow")): + try: + event = json.loads(line) + except ValueError: + event = None + if isinstance(event, dict): + fields = event.get("fields", {}) + if not isinstance(fields, dict) or fields.get("message") not in messages: + continue + value = {} + for key, allowed in (("stage", stages), ("category", categories)): + if fields.get(key) in allowed: + value[key] = fields[key] + for key in ("timed_out", "connect_error", "authorized"): + if isinstance(fields.get(key), bool): + value[key] = fields[key] + if type(fields.get("http_status")) is int and 100 <= fields["http_status"] <= 599: + value["httpStatus"] = fields["http_status"] + if type(fields.get("elapsed_seconds")) is int and 0 <= fields["elapsed_seconds"] <= 999: + value["elapsedSeconds"] = fields["elapsed_seconds"] + if value and value not in facts: + facts.append(value) + continue + if not any(message in line for message in messages): continue value = {} for key, allowed in (("stage", stages), ("category", categories)): diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 9cdbb7dba..b18c5ac21 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -15,6 +15,24 @@ class BootstrapProofTests(unittest.TestCase): + def test_actual_json_tracing_format_is_parsed_without_publishing_other_fields(self): + events = [ + {"message": "SRE authority transport failure", "stage": "registration", + "timed_out": True, "connect_error": False, "token": "do-not-publish"}, + {"message": "SRE authority request denied", "stage": "privacy-review", "http_status": 403}, + {"message": "SRE readiness authority slow", "authorized": True, "elapsed_seconds": 8}, + {"message": "unrelated do-not-publish", "stage": "namespace"}, + {"message": "SRE authority request denied", "stage": "do-not-publish", "http_status": True}, + ] + text = "\n".join(json.dumps({"fields": fields, "span": {"token": "do-not-publish"}}) for fields in events) + facts = router_readiness_facts(text) + self.assertEqual(facts, [ + {"stage": "registration", "timed_out": True, "connect_error": False}, + {"stage": "privacy-review", "httpStatus": 403}, + {"authorized": True, "elapsedSeconds": 8}, + ]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + def test_router_readiness_logs_expose_only_fixed_stage_status_and_timeout_facts(self): text = ( '\x1b[33mSRE authority transport failure\x1b[0m stage="registration" timed_out=true connect_error=false token=do-not-publish\n' From 3f70e35fd8524c03299ad359450e5b0bed6cf8fe Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 16:58:01 +0200 Subject: [PATCH 34/62] test(e2e): distinguish router startup from readiness checks Keep all log data private while reporting Pod restart count, exact expected router image/UID/API-enable booleans, log encoding counts and only checked-in source coordinates for recognized startup messages. Accept nested and flattened JSON tracing. No production behavior or readiness changes; 71 Python regressions pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../sre_authority/bootstrap_diagnostics.py | 36 +++++++++++++++++-- .../e2e/sre_authority/bootstrap_probe_test.py | 18 +++++++++- tests/e2e/sre_authority/common.py | 9 ++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 5f610dca6..d5eccfcc8 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -28,7 +28,7 @@ def probe_command_result(code, output): category = "succeeded" elif "executable file not found" in output: category = "executable-not-found" - elif code == 1 and not output.strip(): + elif code == 1 and output.strip() in ("", "command terminated with exit code 1"): category = "probe-not-ready" return {"exitCode": code, "category": category} @@ -49,7 +49,7 @@ def router_readiness_facts(text): except ValueError: event = None if isinstance(event, dict): - fields = event.get("fields", {}) + fields = event.get("fields", event) if not isinstance(fields, dict) or fields.get("message") not in messages: continue value = {} @@ -88,6 +88,38 @@ def router_readiness_facts(text): return facts[:16] +def router_log_summary(text, root): + paths = [ + root / "inference-router/src/main.rs", + root / "inference-router/src/routes/mod.rs", + root / "inference-router/src/sre_proxy/mod.rs", + root / "inference-router/src/sre_proxy/backend.rs", + ] + sources = [(path, path.read_text().splitlines()) for path in paths] + summary = {"lines": len(text.splitlines()), "jsonEvents": 0, "sourceSites": []} + for line in text.splitlines()[-150:]: + try: + obj = json.loads(line) + except ValueError: + continue + if not isinstance(obj, dict): + continue + summary["jsonEvents"] += 1 + fields = obj.get("fields", obj) + if not isinstance(fields, dict) or not isinstance(fields.get("message"), str): + continue + literal = json.dumps(fields["message"], ensure_ascii=False) + for path, lines in sources: + match = next((number for number, source in enumerate(lines, 1) if literal in source), None) + if match: + site = {"source": str(path.relative_to(root)), "line": match} + if site not in summary["sourceSites"]: + summary["sourceSites"].append(site) + break + summary["sourceSites"] = summary["sourceSites"][:32] + return summary + + def identifier(value): return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index b18c5ac21..e7112ad4a 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import patch -from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, probe_command_result, public_stack_facts, router_readiness_facts +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, probe_command_result, public_stack_facts, router_log_summary, router_readiness_facts from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ @@ -15,6 +15,20 @@ class BootstrapProofTests(unittest.TestCase): + def test_router_startup_summary_reports_only_verified_source_coordinates(self): + root = Path(__file__).resolve().parents[3] + text = "\n".join([ + json.dumps({"fields": {"message": "kars Inference Router starting", "token": "do-not-publish"}}), + json.dumps({"fields": {"message": "do-not-publish"}}), + "do-not-publish plaintext", + ]) + result = router_log_summary(text, root) + self.assertEqual(result["lines"], 3) + self.assertEqual(result["jsonEvents"], 2) + self.assertEqual(len(result["sourceSites"]), 1) + self.assertEqual(result["sourceSites"][0]["source"], "inference-router/src/main.rs") + self.assertNotIn("do-not-publish", json.dumps(result)) + def test_actual_json_tracing_format_is_parsed_without_publishing_other_fields(self): events = [ {"message": "SRE authority transport failure", "stage": "registration", @@ -32,6 +46,7 @@ def test_actual_json_tracing_format_is_parsed_without_publishing_other_fields(se {"authorized": True, "elapsedSeconds": 8}, ]) self.assertNotIn("do-not-publish", json.dumps(facts)) + self.assertEqual(router_readiness_facts(json.dumps(events[0])), [facts[0]]) def test_router_readiness_logs_expose_only_fixed_stage_status_and_timeout_facts(self): text = ( @@ -55,6 +70,7 @@ def test_probe_diagnostics_never_publish_executable_output(self): for code, output, category in ( (127, "exec: executable file not found in $PATH do-not-publish", "executable-not-found"), (1, "", "probe-not-ready"), + (1, "command terminated with exit code 1\n", "probe-not-ready"), (0, "do-not-publish", "succeeded"), (1, "do-not-publish", "unclassified"), ): diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index f58094450..b41f2a50d 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -404,6 +404,7 @@ def diagnostics(self): print("SRE-DIAG", json.dumps({"kind": "Pod", "name": pod["metadata"]["name"], "uid": pod["metadata"]["uid"], "phase": status.get("phase"), "containers": [{"name": container["name"], "ready": container.get("ready"), + "restartCount": container.get("restartCount"), "state": {kind: {key: value.get(key) for key in ("reason", "exitCode") if key in value} for kind, value in container.get("state", {}).items()}} for container in containers]}), flush=True) @@ -415,8 +416,13 @@ def diagnostics(self): print("SRE-DIAG runtime Pod status unavailable", flush=True) def readiness_diagnostics(self, pod): - from .bootstrap_diagnostics import probe_command_result, router_readiness_facts + from .bootstrap_diagnostics import probe_command_result, router_log_summary, router_readiness_facts facts = {"kind": "RouterReadiness", "podUid": pod["metadata"]["uid"]} + router = next(item for item in pod["spec"]["containers"] if item["name"] == "inference-router") + facts["privateApiEnabled"] = any(entry.get("name") == "KARS_SRE_API_ENABLED" + and entry.get("value") == "true" for entry in router.get("env", [])) + facts["routerUid1001"] = router.get("securityContext", {}).get("runAsUser") == 1001 + facts["expectedImage"] = router.get("image") == "kars-inference-router:e2e" try: for label, executable in (("configuredCommand", "kars-inference-router"), ("absoluteCommand", "/usr/local/bin/kars-inference-router")): @@ -436,6 +442,7 @@ def readiness_diagnostics(self, pod): logs = self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", "--tail=150", timeout=10) facts["authorityChecks"] = router_readiness_facts(logs) + facts["logSummary"] = router_log_summary(logs, self.root) except Exception: facts["authorityChecksUnavailable"] = True print("SRE-DIAG", json.dumps(facts), flush=True) From e243f08a11f0038700c77d2350ea26bd890f76a3 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 17:24:05 +0200 Subject: [PATCH 35/62] fix(sre): trace bounded TLS and authority readiness stages Real Pod evidence confirms zero restarts, UID 1001, enabled private API and the correct checksum-verified binary, but no completed authority checks. Add payload-free debug phases for listener binding, TLS handshake, readiness entry and authority-read progress using the existing router debug target. Retain all TLS/authority decisions and deadlines unchanged. Restore standard same-run build-rust dependency and artifacts for full Kind; remove temporary diagnostic artifact wiring. Python: 71 passed; no local Cargo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 16 ++---------- inference-router/src/sre_proxy/backend.rs | 25 ++++++++++++++++--- inference-router/src/sre_proxy/mod.rs | 24 ++++++++++++++++-- .../sre_authority/bootstrap_diagnostics.py | 14 +++++++---- .../e2e/sre_authority/bootstrap_probe_test.py | 5 ++++ tests/e2e/sre_authority/common.py | 3 +++ 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 882e4b14a..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -602,7 +602,7 @@ jobs: e2e-kind: name: E2E (Kind) - needs: [changes] + needs: [changes, build-rust] # Phase 3 S4: closes the audit gap "make test-e2e is not in CI". # Runs on every push to dev/main, manual dispatch, and PRs that # touch the runtime surface area (controller, router, helm chart, @@ -702,20 +702,8 @@ jobs: if: steps.paths.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kars-binaries-921a38354ff2cdcb363f15974a9d4bfd449641bc - run-id: 34359455786 - github-token: ${{ github.token }} + name: kars-binaries-${{ github.sha }} path: ./bin/ - - name: Verify diagnostic binary production equivalence and checksums - if: steps.paths.outputs.run == 'true' - run: | - git fetch --no-tags --depth=1 origin 921a38354ff2cdcb363f15974a9d4bfd449641bc - changed=$(git diff --name-only 921a38354ff2cdcb363f15974a9d4bfd449641bc HEAD) - if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|\.github/workflows/ci\.yml$|$)'; then - echo "Refusing diagnostic binaries from different production sources" - exit 1 - fi - (cd bin && sha256sum --check SHA256SUMS) - name: chmod binaries if: steps.paths.outputs.run == 'true' # Per-arch layout (./bin/amd64/) matches release-internal.yml's diff --git a/inference-router/src/sre_proxy/backend.rs b/inference-router/src/sre_proxy/backend.rs index 41c582543..71b98230e 100644 --- a/inference-router/src/sre_proxy/backend.rs +++ b/inference-router/src/sre_proxy/backend.rs @@ -54,6 +54,15 @@ fn denied_response(stage: &'static str, status: reqwest::StatusCode) { ); } +fn authority_progress(stage: &'static str, step: &'static str) { + tracing::debug!( + target: "inference_router::sre_proxy", + stage, + step, + "SRE authority progress" + ); +} + pub(super) struct Backend { pub config: Config, client: reqwest::Client, @@ -198,6 +207,9 @@ impl Backend { } async fn metadata_json(&self, path: &str, stage: &'static str) -> Result { + authority_progress(stage, "token"); + let token = self.bearer().await?; + authority_progress(stage, "request"); let response = self .client .get(format!( @@ -205,7 +217,7 @@ impl Backend { self.config.kube_url.trim_end_matches('/'), path )) - .bearer_auth(self.bearer().await?) + .bearer_auth(token) .header("accept", "application/json") .send() .await @@ -213,14 +225,17 @@ impl Backend { transport_failure(stage, &error); "SRE authority read failed" })?; + authority_progress(stage, "headers"); if !response.status().is_success() { denied_response(stage, response.status()); return Err("SRE authority read denied".into()); } - response + let value = response .json() .await - .map_err(|_| "SRE authority response invalid".into()) + .map_err(|_| "SRE authority response invalid".to_string())?; + authority_progress(stage, "complete"); + Ok(value) } pub(super) async fn authorize(&self) -> Result<(), String> { @@ -299,6 +314,7 @@ impl Backend { } async fn verify_privacy(&self) -> Result<(), String> { + authority_progress("privacy-review", "request"); for review in crate::sre_privacy::secret_access_reviews(&self.config.runtime_namespace) { let response = self .client @@ -324,6 +340,8 @@ impl Backend { .map_err(|_| "SRE privacy authorization response invalid")?; crate::sre_privacy::require_denial(&response)?; } + authority_progress("privacy-review", "complete"); + authority_progress("credential-metadata", "request"); let response = self .client .get(format!( @@ -354,6 +372,7 @@ impl Backend { &metadata, &[self.config.service_account_uid.as_str()], )?; + authority_progress("credential-metadata", "complete"); Ok(()) } diff --git a/inference-router/src/sre_proxy/mod.rs b/inference-router/src/sre_proxy/mod.rs index 55160ad82..247c7ff0d 100644 --- a/inference-router/src/sre_proxy/mod.rs +++ b/inference-router/src/sre_proxy/mod.rs @@ -76,7 +76,16 @@ fn readiness_failure_category(reason: &str) -> &'static str { } } +fn transport_progress(stage: &'static str) { + tracing::debug!( + target: "inference_router::sre_proxy", + stage, + "SRE transport progress" + ); +} + async fn ready(State(proxy): State) -> Response { + transport_progress("readiness-entered"); let Ok(_permit) = proxy.capacity.try_acquire() else { return error( StatusCode::TOO_MANY_REQUESTS, @@ -84,6 +93,7 @@ async fn ready(State(proxy): State) -> Response { ); }; let started = std::time::Instant::now(); + transport_progress("authority-entered"); let authorization = proxy.backend.authorize().await; let elapsed_seconds = started.elapsed().as_secs(); if elapsed_seconds >= 2 { @@ -268,13 +278,19 @@ impl axum::serve::Listener for Listener { loop { match self.tcp.accept().await { Ok((stream, address)) => { - if let Ok(Ok(stream)) = tokio::time::timeout( + transport_progress("tcp-accepted"); + match tokio::time::timeout( std::time::Duration::from_secs(3), self.tls.accept(stream), ) .await { - return (stream, address); + Ok(Ok(stream)) => { + transport_progress("tls-accepted"); + return (stream, address); + } + Ok(Err(_)) => transport_progress("tls-rejected"), + Err(_) => transport_progress("tls-timeout"), } } Err(_) => tokio::time::sleep(std::time::Duration::from_millis(100)).await, @@ -308,6 +324,7 @@ pub async fn start() -> Result>, String> { if std::env::var("KARS_SRE_API_ENABLED").as_deref() != Ok("true") { return Ok(None); } + transport_progress("enabled"); let directory = PathBuf::from(DIRECTORY); let backend = Backend::load(&directory)?; let token = std::fs::read_to_string(directory.join("agent-token")) @@ -315,12 +332,14 @@ pub async fn start() -> Result>, String> { if token.trim().len() != 64 || !token.trim().bytes().all(|b| b.is_ascii_alphanumeric()) { return Err("SRE proxy credential invalid".into()); } + transport_progress("loaded"); let listener = Listener { tcp: TcpListener::bind(("127.0.0.1", PORT)) .await .map_err(|_| "SRE loopback TLS listener unavailable")?, tls: tls(&directory)?, }; + transport_progress("bound"); backend.renew_in_background(); let proxy = Proxy { backend, @@ -329,6 +348,7 @@ pub async fn start() -> Result>, String> { }; let router = app(proxy); Ok(Some(tokio::spawn(async move { + transport_progress("serving"); if axum::serve(listener, router).await.is_err() { tracing::error!("SRE TLS proxy stopped; router must restart"); std::process::exit(1); diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index d5eccfcc8..32e9641c4 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -35,13 +35,17 @@ def probe_command_result(code, output): def router_readiness_facts(text): facts = [] - stages = ("registration", "namespace", "source", "service-account", "privacy-review", "credential-metadata") + stages = ("registration", "namespace", "source", "service-account", "privacy-review", "credential-metadata", + "enabled", "loaded", "bound", "serving", "tcp-accepted", "tls-accepted", "tls-rejected", + "tls-timeout", "readiness-entered", "authority-entered") + steps = ("token", "request", "headers", "complete") categories = ("authority-transport", "authority-denied", "registration-stale", "namespace-claim", "source-identity", "service-account", "privacy-transport", "privacy-denied", "privacy-not-denied", "metadata-transport", "metadata-denied", "legacy-alias", "credential-expired", "unclassified") messages = ("SRE authority transport failure", "SRE authority request denied", - "SRE readiness authority rejected", "SRE readiness authority slow") + "SRE readiness authority rejected", "SRE readiness authority slow", + "SRE transport progress", "SRE authority progress") for line in text.splitlines()[-150:]: line = re.sub(r"\x1b\[[0-9;]*m", "", line) try: @@ -53,7 +57,7 @@ def router_readiness_facts(text): if not isinstance(fields, dict) or fields.get("message") not in messages: continue value = {} - for key, allowed in (("stage", stages), ("category", categories)): + for key, allowed in (("stage", stages), ("category", categories), ("step", steps)): if fields.get(key) in allowed: value[key] = fields[key] for key in ("timed_out", "connect_error", "authorized"): @@ -69,7 +73,7 @@ def router_readiness_facts(text): if not any(message in line for message in messages): continue value = {} - for key, allowed in (("stage", stages), ("category", categories)): + for key, allowed in (("stage", stages), ("category", categories), ("step", steps)): match = re.search(rf'\b{key}="?({"|".join(allowed)})"?(?:\s|$)', line) if match: value[key] = match[1] @@ -85,7 +89,7 @@ def router_readiness_facts(text): value["elapsedSeconds"] = int(elapsed[1]) if value and value not in facts: facts.append(value) - return facts[:16] + return facts[:32] def router_log_summary(text, root): diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index e7112ad4a..cf4da7b69 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -35,6 +35,9 @@ def test_actual_json_tracing_format_is_parsed_without_publishing_other_fields(se "timed_out": True, "connect_error": False, "token": "do-not-publish"}, {"message": "SRE authority request denied", "stage": "privacy-review", "http_status": 403}, {"message": "SRE readiness authority slow", "authorized": True, "elapsed_seconds": 8}, + {"message": "SRE transport progress", "stage": "tls-accepted", "peer": "do-not-publish"}, + {"message": "SRE authority progress", "stage": "registration", + "step": "request", "url": "do-not-publish"}, {"message": "unrelated do-not-publish", "stage": "namespace"}, {"message": "SRE authority request denied", "stage": "do-not-publish", "http_status": True}, ] @@ -44,6 +47,8 @@ def test_actual_json_tracing_format_is_parsed_without_publishing_other_fields(se {"stage": "registration", "timed_out": True, "connect_error": False}, {"stage": "privacy-review", "httpStatus": 403}, {"authorized": True, "elapsedSeconds": 8}, + {"stage": "tls-accepted"}, + {"stage": "registration", "step": "request"}, ]) self.assertNotIn("do-not-publish", json.dumps(facts)) self.assertEqual(router_readiness_facts(json.dumps(events[0])), [facts[0]]) diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index b41f2a50d..9c3588d2c 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -423,6 +423,9 @@ def readiness_diagnostics(self, pod): and entry.get("value") == "true" for entry in router.get("env", [])) facts["routerUid1001"] = router.get("securityContext", {}).get("runAsUser") == 1001 facts["expectedImage"] = router.get("image") == "kars-inference-router:e2e" + facts["progressLoggingEnabled"] = any(entry.get("name") == "RUST_LOG" + and "inference_router=debug" in entry.get("value", "") + for entry in router.get("env", [])) try: for label, executable in (("configuredCommand", "kars-inference-router"), ("absoluteCommand", "/usr/local/bin/kars-inference-router")): From 63f522590c14148022ee35107c67d47698dadf3e Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 18:14:58 +0200 Subject: [PATCH 36/62] test(e2e): isolate private router API transport failures TLS and readiness entry succeed, then the first registration GET stalls. Failure-only diagnostics now outwait the unchanged backend timeout and check TCP to the cluster Service and real API endpoint using the already-loaded stand-in, UID 1001, no mounts/credentials/shared process namespace and UID/RV-fenced ephemeral-container insertion. No production readiness timeout or permission changes. Temporarily reuse source-equivalent SHA256-verified e243f08a CI binaries while all Rust gates still run; restore ordinary dependency before final qualification. Python: 72 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 16 +++++- tests/e2e/sre_authority/common.py | 65 ++++++++++++++++++++++++- tests/e2e/sre_authority/harness_test.py | 39 +++++++++++++++ 3 files changed, 116 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7157bc56c..3a00abcb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -602,7 +602,7 @@ jobs: e2e-kind: name: E2E (Kind) - needs: [changes, build-rust] + needs: [changes] # Phase 3 S4: closes the audit gap "make test-e2e is not in CI". # Runs on every push to dev/main, manual dispatch, and PRs that # touch the runtime surface area (controller, router, helm chart, @@ -702,8 +702,20 @@ jobs: if: steps.paths.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kars-binaries-${{ github.sha }} + name: kars-binaries-2945756ef265d41d984b8a4b1615b6a1a1291d53 + run-id: 34370014115 + github-token: ${{ github.token }} path: ./bin/ + - name: Verify diagnostic binary production equivalence and checksums + if: steps.paths.outputs.run == 'true' + run: | + git fetch --no-tags --depth=1 origin 2945756ef265d41d984b8a4b1615b6a1a1291d53 + changed=$(git diff --name-only 2945756ef265d41d984b8a4b1615b6a1a1291d53 HEAD) + if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|\.github/workflows/ci\.yml$|$)'; then + echo "Refusing diagnostic binaries from different production sources" + exit 1 + fi + (cd bin && sha256sum --check SHA256SUMS) - name: chmod binaries if: steps.paths.outputs.run == 'true' # Per-arch layout (./bin/amd64/) matches release-internal.yml's diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 9c3588d2c..a72f0671a 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -374,7 +374,7 @@ def passed(self, message): def diagnostics(self): from .bootstrap_diagnostics import failure_facts - self.deadline = max(self.deadline, time.monotonic() + 50) + self.deadline = max(self.deadline, time.monotonic() + 90) # Status and identities only; never dump Secret bodies or whole Pods. for kind, name, namespace in [("karssreregistrations.kars.azure.com", "canonical", None), ("karssandbox", "sre", SYSTEM), ("deployment", "sre", RUNTIME)]: @@ -426,6 +426,10 @@ def readiness_diagnostics(self, pod): facts["progressLoggingEnabled"] = any(entry.get("name") == "RUST_LOG" and "inference_router=debug" in entry.get("value", "") for entry in router.get("env", [])) + try: + facts["apiConnectivity"] = self.connectivity_diagnostics(pod) + except Exception as error: + facts["connectivityDiagnosticError"] = type(error).__name__ try: for label, executable in (("configuredCommand", "kars-inference-router"), ("absoluteCommand", "/usr/local/bin/kars-inference-router")): @@ -436,7 +440,7 @@ def readiness_diagnostics(self, pod): "--", "cat", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", timeout=10) context = ssl.create_default_context(cadata=ca) with self.port_forward(pod["metadata"]["name"]) as port, \ - self.httpx.Client(verify=context, timeout=5, trust_env=False) as client: + self.httpx.Client(verify=context, timeout=25, trust_env=False) as client: response = client.get(f"https://127.0.0.1:{port}/readyz") facts["verifiedLoopbackTlsStatus"] = response.status_code except Exception as error: @@ -450,6 +454,63 @@ def readiness_diagnostics(self, pod): facts["authorityChecksUnavailable"] = True print("SRE-DIAG", json.dumps(facts), flush=True) + def connectivity_diagnostics(self, pod): + import ipaddress + service = self.get("service", "kubernetes", "default") + endpoint = self.get("endpoints", "kubernetes", "default") + service_ip = service["spec"]["clusterIP"] + service_port = next(port["port"] for port in service["spec"]["ports"] if port["name"] == "https") + subset = endpoint["subsets"][0] + endpoint_ip = subset["addresses"][0]["ip"] + endpoint_port = next(port["port"] for port in subset["ports"] if port["name"] == "https") + require(all(ipaddress.ip_address(address).is_private for address in (service_ip, endpoint_ip)) + and all(type(port) is int and 0 < port < 65536 for port in (service_port, endpoint_port)), + "Connectivity diagnostic refuses non-private or invalid API targets") + current = self.get("pod", pod["metadata"]["name"], RUNTIME) + require(current["metadata"]["uid"] == pod["metadata"]["uid"], + "Diagnostic Pod changed before connectivity inspection") + require(current["spec"].get("automountServiceAccountToken") is False + and current["spec"].get("shareProcessNamespace") is not True, + "Connectivity diagnostic requires isolated processes and no ambient token") + name = "sre-e2e-network-diagnostic" + existing = current["spec"].get("ephemeralContainers", []) + require(not any(item["name"] == name for item in existing), "Diagnostic container name is already occupied") + script = """ +if ! command -v timeout >/dev/null || ! command -v bash >/dev/null; then + printf '{"available":false}\\n'; exit 0 +fi +timeout 6 bash -c 'exec 3<>/dev/tcp/"$1"/"$2"' sre-tcp "$1" "$2" >/dev/null 2>&1 +service=$? +timeout 6 bash -c 'exec 3<>/dev/tcp/"$1"/"$2"' sre-tcp "$3" "$4" >/dev/null 2>&1 +endpoint=$? +printf '{"available":true,"serviceExit":%s,"endpointExit":%s}\\n' "$service" "$endpoint" +""" + probe = {"name": name, "image": STANDIN, "imagePullPolicy": "IfNotPresent", + "command": ["/bin/sh", "-c", script, "sre-connectivity", service_ip, str(service_port), + endpoint_ip, str(endpoint_port)], + "securityContext": {"runAsUser": 1001, "runAsNonRoot": True, + "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, + "capabilities": {"drop": ["ALL"]}}} + self.api("PATCH", f"/api/v1/namespaces/{RUNTIME}/pods/{pod['metadata']['name']}/ephemeralcontainers", + body={"metadata": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}, + "spec": {"ephemeralContainers": existing + [probe]}}, status=200) + def completed(): + current = self.get("pod", pod["metadata"]["name"], RUNTIME) + require(current and current["metadata"]["uid"] == pod["metadata"]["uid"], + "Diagnostic Pod changed during connectivity inspection") + return any(item.get("name") == name and "terminated" in item.get("state", {}) + for item in current.get("status", {}).get("ephemeralContainerStatuses", [])) + self.poll("bounded UID-1001 API TCP probes", completed, seconds=20, interval=0.5) + result = json.loads(self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", name, "--tail=5")) + require(isinstance(result, dict) and type(result.get("available")) is bool + and set(result) == ({"available", "serviceExit", "endpointExit"} + if result["available"] else {"available"}) + and all(type(value) is int and 0 <= value <= 255 + for key, value in result.items() if key != "available"), + "Connectivity diagnostic produced an unexpected result") + return result + def close(self): for process in self.processes: process.terminate() diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 0380450e9..77cf1836c 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -34,6 +34,45 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_connectivity_diagnostic_is_uid_fenced_nonprivileged_and_has_no_credentials(self): + h = Harness.__new__(Harness) + pod = {"metadata": {"name": "sre-probe", "uid": "pod-uid", "resourceVersion": "17"}, + "spec": {"automountServiceAccountToken": False}, + "status": {"ephemeralContainerStatuses": [{"name": "sre-e2e-network-diagnostic", + "state": {"terminated": {"exitCode": 0}}}]}} + def get(kind, _name, _namespace): + if kind == "service": + return {"spec": {"clusterIP": "10.96.0.1", "ports": [{"name": "https", "port": 443}]}} + if kind == "endpoints": + return {"subsets": [{"addresses": [{"ip": "172.18.0.2"}], + "ports": [{"name": "https", "port": 6443}]}]} + return pod + writes = [] + h.get = get + h.api = lambda method, path, **kwargs: writes.append((method, path, kwargs)) + h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) + h.k = lambda *_args: '{"available":true,"serviceExit":0,"endpointExit":0}' + self.assertEqual(h.connectivity_diagnostics(pod), {"available": True, "serviceExit": 0, "endpointExit": 0}) + self.assertEqual(len(writes), 1) + method, path, args = writes[0] + self.assertEqual(method, "PATCH") + self.assertTrue(path.endswith("/ephemeralcontainers")) + body = args["body"] + self.assertEqual(body["metadata"], {"uid": "pod-uid", "resourceVersion": "17"}) + probe = body["spec"]["ephemeralContainers"][0] + self.assertEqual(probe["securityContext"]["runAsUser"], 1001) + self.assertFalse(probe["securityContext"]["allowPrivilegeEscalation"]) + self.assertEqual(probe["securityContext"]["capabilities"], {"drop": ["ALL"]}) + for field in ("targetContainerName", "volumeMounts", "env", "envFrom"): + self.assertNotIn(field, probe) + for key, value in (("automountServiceAccountToken", True), ("shareProcessNamespace", True)): + old = copy.deepcopy(pod["spec"]) + pod["spec"][key] = value + with self.assertRaises(AssertionError): + h.connectivity_diagnostics(pod) + pod["spec"] = old + self.assertEqual(len(writes), 1) + def test_blocked_authority_diagnostics_report_source_coordinates_not_status_detail(self): root = Path(__file__).resolve().parents[3] cases = [ From cf2f2745d705fba5976299aadbdf0c0326147f27 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 18:34:30 +0200 Subject: [PATCH 37/62] test(e2e): compare same-node API connectivity without credentials Real UID-1001 probes time out to both API Service and endpoint while loopback TLS returns 503 after the unchanged backend 20-second timeout. Add a same-node control Pod with the same seccomp/container context, no credentials, no mounts, no shared PID namespace, create-only ownership and UID/RV-fenced cleanup. Record only typed UID/connectivity results and known CNI/SRE-rule booleans. Preserve all production guards. Python: 74 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/common.py | 66 +++++++++++++++++++++++-- tests/e2e/sre_authority/harness_test.py | 36 +++++++++++++- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index a72f0671a..0bec34b0e 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -3,6 +3,7 @@ import base64 import contextlib +import copy import hashlib import inspect import json @@ -430,6 +431,22 @@ def readiness_diagnostics(self, pod): facts["apiConnectivity"] = self.connectivity_diagnostics(pod) except Exception as error: facts["connectivityDiagnosticError"] = type(error).__name__ + try: + source = self.get("karssandbox", "sre", SYSTEM) + policy = self.get("networkpolicy", "sandbox-policy", RUNTIME) + service = self.get("service", "kubernetes", "default") + service_ip = service["spec"]["clusterIP"] + rules = policy.get("spec", {}).get("egress", []) if policy else [] + facts["sourceLabelSre"] = source.get("metadata", {}).get("labels", {}).get("kars.azure.com/role") == "sre" + facts["explicitApiServiceRule"] = any( + any(peer.get("ipBlock", {}).get("cidr") == f"{service_ip}/32" for peer in rule.get("to", [])) + and any(port.get("port") == 443 and port.get("protocol", "TCP") == "TCP" + for port in rule.get("ports", [])) for rule in rules) + agents = json.loads(self.k("get", "daemonsets", "-n", "kube-system", "-o", "json"))["items"] + facts["knownNetworkAgents"] = sorted(agent["metadata"]["name"] for agent in agents + if agent["metadata"]["name"] in ("kindnet", "cilium", "calico-node", "antrea-agent", "kube-flannel-ds")) + except Exception as error: + facts["networkPolicyDiagnosticError"] = type(error).__name__ try: for label, executable in (("configuredCommand", "kars-inference-router"), ("absoluteCommand", "/usr/local/bin/kars-inference-router")): @@ -476,14 +493,16 @@ def connectivity_diagnostics(self, pod): existing = current["spec"].get("ephemeralContainers", []) require(not any(item["name"] == name for item in existing), "Diagnostic container name is already occupied") script = """ -if ! command -v timeout >/dev/null || ! command -v bash >/dev/null; then +if ! command -v timeout >/dev/null || ! command -v bash >/dev/null || ! command -v id >/dev/null; then printf '{"available":false}\\n'; exit 0 fi +uid=false +[ "$(id -u)" = 1001 ] && uid=true timeout 6 bash -c 'exec 3<>/dev/tcp/"$1"/"$2"' sre-tcp "$1" "$2" >/dev/null 2>&1 service=$? timeout 6 bash -c 'exec 3<>/dev/tcp/"$1"/"$2"' sre-tcp "$3" "$4" >/dev/null 2>&1 endpoint=$? -printf '{"available":true,"serviceExit":%s,"endpointExit":%s}\\n' "$service" "$endpoint" +printf '{"available":true,"uidMatches1001":%s,"serviceExit":%s,"endpointExit":%s}\\n' "$uid" "$service" "$endpoint" """ probe = {"name": name, "image": STANDIN, "imagePullPolicy": "IfNotPresent", "command": ["/bin/sh", "-c", script, "sre-connectivity", service_ip, str(service_port), @@ -502,15 +521,52 @@ def completed(): return any(item.get("name") == name and "terminated" in item.get("state", {}) for item in current.get("status", {}).get("ephemeralContainerStatuses", [])) self.poll("bounded UID-1001 API TCP probes", completed, seconds=20, interval=0.5) - result = json.loads(self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", name, "--tail=5")) + result = self.parse_connectivity_result(self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", name, "--tail=5")) + try: + result["sameNodeControl"] = self.control_connectivity_diagnostics(pod, probe) + except Exception as error: + result["controlDiagnosticError"] = type(error).__name__ + return result + + @staticmethod + def parse_connectivity_result(output): + result = json.loads(output) require(isinstance(result, dict) and type(result.get("available")) is bool - and set(result) == ({"available", "serviceExit", "endpointExit"} + and set(result) == ({"available", "uidMatches1001", "serviceExit", "endpointExit"} if result["available"] else {"available"}) + and (not result["available"] or type(result.get("uidMatches1001")) is bool) and all(type(value) is int and 0 <= value <= 255 - for key, value in result.items() if key != "available"), + for key, value in result.items() if key not in ("available", "uidMatches1001")), "Connectivity diagnostic produced an unexpected result") return result + def control_connectivity_diagnostics(self, pod, container): + name = "sre-e2e-api-connectivity" + require(pod["spec"].get("nodeName"), "Connectivity comparison requires the actual sandbox node") + created = self.create({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "namespace": TENANT}, + "spec": {"nodeName": pod["spec"]["nodeName"], "automountServiceAccountToken": False, + "restartPolicy": "Never", "securityContext": copy.deepcopy(pod["spec"].get("securityContext", {})), + "containers": [copy.deepcopy(container)]}}) + uid = created["metadata"]["uid"] + try: + def completed(): + current = self.get("pod", name, TENANT) + require(current and current["metadata"]["uid"] == uid, "Connectivity control Pod was replaced") + return current.get("status", {}).get("phase") in ("Succeeded", "Failed") + self.poll("same-node credential-free TCP comparison", completed, seconds=25, interval=0.5) + return self.parse_connectivity_result( + self.k("logs", "-n", TENANT, name, "-c", container["name"], "--tail=5")) + finally: + try: + current = self.get("pod", name, TENANT) + require(current and current["metadata"]["uid"] == uid, "Connectivity control cleanup identity changed") + self.api("DELETE", f"/api/v1/namespaces/{TENANT}/pods/{name}", body={ + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, + status=(200, 202)) + except Exception: + print("SRE-DIAG owned connectivity control cleanup unavailable", flush=True) + def close(self): for process in self.processes: process.terminate() diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 77cf1836c..1fd628247 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -51,8 +51,11 @@ def get(kind, _name, _namespace): h.get = get h.api = lambda method, path, **kwargs: writes.append((method, path, kwargs)) h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) - h.k = lambda *_args: '{"available":true,"serviceExit":0,"endpointExit":0}' - self.assertEqual(h.connectivity_diagnostics(pod), {"available": True, "serviceExit": 0, "endpointExit": 0}) + h.k = lambda *_args: '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}' + h.control_connectivity_diagnostics = lambda *_args: {"available": False} + self.assertEqual(h.connectivity_diagnostics(pod), { + "available": True, "uidMatches1001": True, "serviceExit": 0, "endpointExit": 0, + "sameNodeControl": {"available": False}}) self.assertEqual(len(writes), 1) method, path, args = writes[0] self.assertEqual(method, "PATCH") @@ -73,6 +76,35 @@ def get(kind, _name, _namespace): pod["spec"] = old self.assertEqual(len(writes), 1) + def test_connectivity_result_rejects_arbitrary_output_or_untyped_fields(self): + for value in ({"available": True}, {"available": 1}, + {"available": False, "data": MARKER}, + {"available": True, "uidMatches1001": True, "serviceExit": "0", "endpointExit": 0}): + with self.assertRaises(AssertionError): + Harness.parse_connectivity_result(json.dumps(value)) + self.assertEqual(Harness.parse_connectivity_result('{"available":false}'), {"available": False}) + + def test_same_node_control_creates_only_unprivileged_owned_pod_and_uid_fenced_cleanup(self): + h = Harness.__new__(Harness) + pod = {"spec": {"nodeName": "sandbox-worker", "securityContext": {"runAsNonRoot": True}}} + container = {"name": "network", "image": "kars-sandbox-e2e:dev", + "securityContext": {"runAsUser": 1001, "capabilities": {"drop": ["ALL"]}}} + seen, deleted = [], [] + current = {"metadata": {"uid": "control-uid", "resourceVersion": "7"}, "status": {"phase": "Succeeded"}} + h.create = lambda obj: seen.append(obj) or current + h.get = lambda *_args: current + h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) + h.k = lambda *_args: '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}' + h.api = lambda method, path, **kwargs: deleted.append((method, path, kwargs)) + h.control_connectivity_diagnostics(pod, container) + self.assertEqual(len(seen), 1) + self.assertEqual(seen[0]["spec"]["nodeName"], "sandbox-worker") + self.assertFalse(seen[0]["spec"]["automountServiceAccountToken"]) + self.assertNotIn("shareProcessNamespace", seen[0]["spec"]) + self.assertNotIn("volumes", seen[0]["spec"]) + self.assertEqual(deleted[0][0], "DELETE") + self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "control-uid", "resourceVersion": "7"}) + def test_blocked_authority_diagnostics_report_source_coordinates_not_status_detail(self): root = Path(__file__).resolve().parents[3] cases = [ From 4f6e95af638296b929b8308efa337c9f281e61be Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 18:59:53 +0200 Subject: [PATCH 38/62] test(e2e): inspect exact Pod firewall state without mutations Same-node/no-credential control connectivity succeeds while real UID-1001 TCP fails only inside the SRE Pod. Read nft/legacy saved tables through the already-owned Kind node, fencing node label, CRI sandbox UID and live PID before/after namespace inspection. Publish only table/policy/action/owner booleans and counters, never raw rules/comments/addresses; grant no capabilities and alter no firewall or production guards. Python: 76 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../sre_authority/bootstrap_diagnostics.py | 30 ++++++++++++++ .../e2e/sre_authority/bootstrap_probe_test.py | 17 +++++++- tests/e2e/sre_authority/common.py | 40 +++++++++++++++++++ tests/e2e/sre_authority/harness_test.py | 14 +++++++ 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 32e9641c4..7104b1634 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -124,6 +124,36 @@ def router_log_summary(text, root): return summary +def firewall_summary(text): + result = {"policies": [], "rules": []} + table = None + for line in text.splitlines(): + if line.startswith("*"): + table = line[1:] if line[1:] in ("filter", "nat", "raw", "mangle", "security") else None + if not table: + continue + policy = re.fullmatch(r":(INPUT|OUTPUT|FORWARD) (ACCEPT|DROP) \[([0-9]+):[0-9]+\]", line) + if policy: + result["policies"].append({"table": table, "chain": policy[1], + "policy": policy[2], "packets": int(policy[3])}) + match = re.match(r"(?:\[([0-9]+):[0-9]+\] )?-A ([A-Za-z0-9_-]+) ", line) + if not match: + continue + owner = re.search(r"(! )?--uid-owner (1000|1001)(?:\s|$)", line) + action = re.search(r" -j (ACCEPT|DROP|REJECT|REDIRECT|RETURN)(?:\s|$)", line) + result["rules"].append({ + "table": table, "chain": match[2] if match[2] in ("INPUT", "OUTPUT", "FORWARD") else "custom", + "packets": int(match[1]) if match[1] else None, + "action": action[1] if action else "jump-or-other", + "owner": int(owner[2]) if owner else None, + "ownerNegated": bool(owner and owner[1]), + "loopback": bool(re.search(r"(?: -[io] lo)(?:\s|$)", line)), + "established": "--ctstate" in line and "ESTABLISHED" in line, + }) + result["rules"] = result["rules"][:40] + return result + + def identifier(value): return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index cf4da7b69..3bbcd33f5 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import patch -from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, object_status, policy_status, probe_command_result, public_stack_facts, router_log_summary, router_readiness_facts +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, firewall_summary, object_status, policy_status, probe_command_result, public_stack_facts, router_log_summary, router_readiness_facts from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ @@ -15,6 +15,21 @@ class BootstrapProofTests(unittest.TestCase): + def test_firewall_summary_does_not_publish_rules_comments_addresses_or_unknown_chains(self): + text = """*filter +:OUTPUT ACCEPT [12:640] +[4:240] -A OUTPUT -m owner --uid-owner 1000 -j DROP +[3:120] -A OUTPUT -m owner ! --uid-owner 1001 -o lo -j ACCEPT +[2:80] -A do-not-publish -s do-not-publish -m comment --comment do-not-publish -j DROP +COMMIT +""" + value = firewall_summary(text) + self.assertEqual(value["policies"], [{"table": "filter", "chain": "OUTPUT", "policy": "ACCEPT", "packets": 12}]) + self.assertEqual(value["rules"][0]["owner"], 1000) + self.assertTrue(value["rules"][1]["ownerNegated"]) + self.assertEqual(value["rules"][2]["chain"], "custom") + self.assertNotIn("do-not-publish", json.dumps(value)) + def test_router_startup_summary_reports_only_verified_source_coordinates(self): root = Path(__file__).resolve().parents[3] text = "\n".join([ diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 0bec34b0e..239f1e19e 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -445,8 +445,16 @@ def readiness_diagnostics(self, pod): agents = json.loads(self.k("get", "daemonsets", "-n", "kube-system", "-o", "json"))["items"] facts["knownNetworkAgents"] = sorted(agent["metadata"]["name"] for agent in agents if agent["metadata"]["name"] in ("kindnet", "cilium", "calico-node", "antrea-agent", "kube-flannel-ds")) + facts["minimalKindnetImage"] = any( + container.get("image", "").removeprefix("docker.io/").startswith("kindest/kindnetd:") + for agent in agents if agent["metadata"]["name"] == "kindnet" + for container in agent.get("spec", {}).get("template", {}).get("spec", {}).get("containers", [])) except Exception as error: facts["networkPolicyDiagnosticError"] = type(error).__name__ + try: + facts["firewall"] = self.firewall_diagnostics(pod) + except Exception as error: + facts["firewallDiagnosticError"] = type(error).__name__ try: for label, executable in (("configuredCommand", "kars-inference-router"), ("absoluteCommand", "/usr/local/bin/kars-inference-router")): @@ -471,6 +479,38 @@ def readiness_diagnostics(self, pod): facts["authorityChecksUnavailable"] = True print("SRE-DIAG", json.dumps(facts), flush=True) + def firewall_diagnostics(self, pod): + from .bootstrap_diagnostics import firewall_summary + node = pod["spec"].get("nodeName") + require(node in ("kars-e2e-worker", "kars-e2e-control-plane") + and not pod["spec"].get("hostNetwork"), "Firewall diagnostic requires the owned Kind Pod network") + cluster = self.run(["docker", "inspect", node, "--format", + '{{index .Config.Labels "io.x-k8s.kind.cluster"}}'], timeout=10).strip() + require(cluster == "kars-e2e", "Firewall diagnostic refuses a foreign node") + items = json.loads(self.run(["docker", "exec", node, "crictl", "pods", + "--label", f'io.kubernetes.pod.uid={pod["metadata"]["uid"]}', + "-o", "json"], timeout=10))["items"] + matches = [item for item in items if item.get("metadata", {}).get("uid") == pod["metadata"]["uid"] + and item["metadata"].get("namespace") == RUNTIME and item.get("state") == "SANDBOX_READY"] + require(len(matches) == 1 and re.fullmatch(r"[0-9a-f]{64}", matches[0]["id"]), + "Firewall diagnostic could not identify the exact live Pod sandbox") + sandbox = matches[0]["id"] + def identity(): + obj = json.loads(self.run(["docker", "exec", node, "crictl", "inspectp", "-o", "json", sandbox], timeout=10)) + require(obj.get("status", {}).get("metadata", {}).get("uid") == pod["metadata"]["uid"], + "Firewall diagnostic sandbox UID changed") + pid = obj.get("info", {}).get("pid") + require(type(pid) is int and 0 < pid < 2**31, "Firewall diagnostic sandbox PID is invalid") + return pid + pid = identity() + facts = {} + for backend in ("nft", "legacy"): + output = self.run(["docker", "exec", node, "nsenter", "--target", str(pid), "--net", "--", + f"iptables-{backend}-save", "-c"], timeout=10, expected=None) + facts[backend] = firewall_summary(output.stdout) if output.returncode == 0 else {"available": False} + require(identity() == pid, "Firewall diagnostic sandbox changed during the read") + return facts + def connectivity_diagnostics(self, pod): import ipaddress service = self.get("service", "kubernetes", "default") diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 1fd628247..753d8fd52 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -34,6 +34,20 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_firewall_diagnostic_requires_owned_node_before_readonly_namespace_entry(self): + h = Harness.__new__(Harness) + calls = [] + h.run = lambda args, **_kwargs: calls.append(args) or "foreign-cluster" + pod = {"metadata": {"uid": "pod-uid"}, "spec": {"nodeName": "kars-e2e-worker"}} + with self.assertRaises(AssertionError): + h.firewall_diagnostics(pod) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][:3], ["docker", "inspect", "kars-e2e-worker"]) + pod["spec"]["hostNetwork"] = True + with self.assertRaises(AssertionError): + h.firewall_diagnostics(pod) + self.assertEqual(len(calls), 1) + def test_connectivity_diagnostic_is_uid_fenced_nonprivileged_and_has_no_credentials(self): h = Harness.__new__(Harness) pod = {"metadata": {"name": "sre-probe", "uid": "pod-uid", "resourceVersion": "17"}, From e2f6801226c1d5d182ec0819debd49e98088fcfd Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 19:30:50 +0200 Subject: [PATCH 39/62] test(e2e): isolate guard backend effects in owned empty Pods Read-only saved tables show ACCEPT policies and zero UID-1000 rule hits despite UID-1001 transport failure. Compare exact-source full guard, filter-only (retaining every agent DROP restriction), and legacy-backend full guard in new no-credential/no-mount same-node controls. Never alter the SRE Pod's guard; reject unreviewed command/image/extra inputs and fence cleanup by UID/RV. Missing legacy tools are explicitly unavailable, not accepted. Python: 79 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/common.py | 32 ++++++++++--- tests/e2e/sre_authority/harness_test.py | 3 +- .../e2e/sre_authority/network_diagnostics.py | 39 ++++++++++++++++ .../sre_authority/network_diagnostics_test.py | 46 +++++++++++++++++++ 4 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/sre_authority/network_diagnostics.py create mode 100644 tests/e2e/sre_authority/network_diagnostics_test.py diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index 239f1e19e..e6a479d71 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -566,6 +566,12 @@ def completed(): result["sameNodeControl"] = self.control_connectivity_diagnostics(pod, probe) except Exception as error: result["controlDiagnosticError"] = type(error).__name__ + result["guardControls"] = {} + for variant in ("full", "filter-only", "legacy-full"): + try: + result["guardControls"][variant] = self.control_connectivity_diagnostics(pod, probe, variant) + except Exception as error: + result["guardControls"][variant] = {"diagnosticError": type(error).__name__} return result @staticmethod @@ -580,20 +586,32 @@ def parse_connectivity_result(output): "Connectivity diagnostic produced an unexpected result") return result - def control_connectivity_diagnostics(self, pod, container): - name = "sre-e2e-api-connectivity" + def control_connectivity_diagnostics(self, pod, container, variant=None): + from .network_diagnostics import guard_variant + name = "sre-e2e-api-connectivity" + (f"-{variant}" if variant else "") require(pod["spec"].get("nodeName"), "Connectivity comparison requires the actual sandbox node") - created = self.create({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": name, "namespace": TENANT}, - "spec": {"nodeName": pod["spec"]["nodeName"], "automountServiceAccountToken": False, - "restartPolicy": "Never", "securityContext": copy.deepcopy(pod["spec"].get("securityContext", {})), - "containers": [copy.deepcopy(container)]}}) + spec = {"nodeName": pod["spec"]["nodeName"], "automountServiceAccountToken": False, + "restartPolicy": "Never", "securityContext": copy.deepcopy(pod["spec"].get("securityContext", {})), + "containers": [copy.deepcopy(container)]} + if variant: + spec["initContainers"] = [guard_variant(self.root, pod, variant)] + created = self.create({"apiVersion": "v1", "kind": "Pod", + "metadata": {"name": name, "namespace": TENANT}, "spec": spec}) uid = created["metadata"]["uid"] try: def completed(): current = self.get("pod", name, TENANT) require(current and current["metadata"]["uid"] == uid, "Connectivity control Pod was replaced") - return current.get("status", {}).get("phase") in ("Succeeded", "Failed") + status = current.get("status", {}) + return status.get("phase") in ("Succeeded", "Failed") or any( + item.get("state", {}).get("terminated", {}).get("exitCode", 0) != 0 + for item in status.get("initContainerStatuses", [])) self.poll("same-node credential-free TCP comparison", completed, seconds=25, interval=0.5) + current = self.get("pod", name, TENANT) + for item in current.get("status", {}).get("initContainerStatuses", []): + code = item.get("state", {}).get("terminated", {}).get("exitCode", 0) + if code != 0: + return {"available": False, "initExit": code} return self.parse_connectivity_result( self.k("logs", "-n", TENANT, name, "-c", container["name"], "--tail=5")) finally: diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 753d8fd52..cd81aaad5 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -69,7 +69,8 @@ def get(kind, _name, _namespace): h.control_connectivity_diagnostics = lambda *_args: {"available": False} self.assertEqual(h.connectivity_diagnostics(pod), { "available": True, "uidMatches1001": True, "serviceExit": 0, "endpointExit": 0, - "sameNodeControl": {"available": False}}) + "sameNodeControl": {"available": False}, + "guardControls": {variant: {"available": False} for variant in ("full", "filter-only", "legacy-full")}}) self.assertEqual(len(writes), 1) method, path, args = writes[0] self.assertEqual(method, "PATCH") diff --git a/tests/e2e/sre_authority/network_diagnostics.py b/tests/e2e/sre_authority/network_diagnostics.py new file mode 100644 index 000000000..91950ca4c --- /dev/null +++ b/tests/e2e/sre_authority/network_diagnostics.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Throwaway no-credential network controls, never mutations of the SRE guard.""" + +import copy +import json +import re + +from .common import STANDIN, require + + +def guard_variant(root, pod, variant): + require(variant in ("full", "filter-only", "legacy-full"), "Unknown guard comparison") + source = (root / "controller/src/reconciler/pod_spec.rs").read_text() + body = source.split("pub(crate) fn build_egress_guard_command", 1)[1].split("\n cmd\n", 1)[0] + literals = re.findall(r'cmd\.push_str\(\s*("(?:\\.|[^"\\])*")\s*\)', body) + require(len(literals) == 8, "Guard comparison requires the exact reviewed command structure") + expected = "".join(json.loads(literal) for literal in literals) + require(not any(char in expected for char in ("$", "`", ";", "\n")), + "Guard comparison refuses dynamic shell constructs") + matches = [item for item in pod["spec"].get("initContainers", []) if item["name"] == "egress-guard"] + require(len(matches) == 1, "Guard comparison requires exactly one actual egress guard") + original = matches[0] + require(original["image"] == STANDIN and original["command"] == ["sh", "-c", expected] + and not any(original.get(key) for key in ("env", "envFrom", "volumeMounts")), + "Actual guard differs from the reviewed source or contains extra inputs") + script = expected + if variant == "filter-only": + script = " && ".join(part for part in script.split(" && ") if not part.startswith("iptables -t nat ")) + require("iptables -A OUTPUT -m owner --uid-owner 1000 -j DROP" in script, + "Filter comparison must retain the full agent DROP boundary") + elif variant == "legacy-full": + script = re.sub(r"\biptables(?= )", "iptables-legacy", script) + script = "command -v iptables-legacy >/dev/null || exit 42; " + script + init = {key: copy.deepcopy(value) for key, value in original.items() + if key in ("name", "image", "imagePullPolicy", "command", "securityContext", "resources")} + init["command"] = ["sh", "-c", script] + return init diff --git a/tests/e2e/sre_authority/network_diagnostics_test.py b/tests/e2e/sre_authority/network_diagnostics_test.py new file mode 100644 index 000000000..36b5c116e --- /dev/null +++ b/tests/e2e/sre_authority/network_diagnostics_test.py @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +from pathlib import Path +import re +import unittest + +from sre_authority.common import STANDIN +from sre_authority.network_diagnostics import guard_variant + + +class NetworkDiagnosticTests(unittest.TestCase): + def fixture(self): + root = Path(__file__).resolve().parents[3] + source = (root / "controller/src/reconciler/pod_spec.rs").read_text() + body = source.split("pub(crate) fn build_egress_guard_command", 1)[1].split("\n cmd\n", 1)[0] + expected = "".join(json.loads(value) for value in re.findall(r'cmd\.push_str\(\s*("(?:\\.|[^"\\])*")\s*\)', body)) + return root, {"spec": {"initContainers": [{"name": "egress-guard", "image": STANDIN, + "command": ["sh", "-c", expected], "securityContext": {"runAsUser": 0}}]}} + + def test_full_control_reproduces_actual_guard_without_mutating_it(self): + root, pod = self.fixture() + original = copy.deepcopy(pod) + self.assertEqual(guard_variant(root, pod, "full"), pod["spec"]["initContainers"][0]) + self.assertEqual(pod, original) + + def test_filter_control_preserves_uid_drop_and_legacy_control_preserves_every_rule(self): + root, pod = self.fixture() + filtered = guard_variant(root, pod, "filter-only")["command"][2] + self.assertIn("--uid-owner 1000 -j DROP", filtered) + self.assertNotIn("-t nat", filtered) + legacy = guard_variant(root, pod, "legacy-full")["command"][2] + expected = pod["spec"]["initContainers"][0]["command"][2] + self.assertEqual(legacy.split("; ", 1)[1], expected.replace("iptables ", "iptables-legacy ")) + self.assertIn("|| exit 42", legacy) + + def test_unreviewed_command_image_or_extra_mount_is_never_executed(self): + root, pod = self.fixture() + for key, value in (("image", "other-image"), ("command", ["sh", "-c", "unreviewed"]), + ("volumeMounts", [{"name": "private"}]), ("envFrom", [{"secretRef": {"name": "private"}}])): + changed = copy.deepcopy(pod) + changed["spec"]["initContainers"][0][key] = value + with self.assertRaises(AssertionError): + guard_variant(root, changed, "full") From e4aaf00bd2baa11b6d0d4fa52f95b6b48ea839db Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 19:52:39 +0200 Subject: [PATCH 40/62] test(e2e): compare isolated network policy enforcement Exact-source full nft, filter-only and full legacy guard controls all connect, so do not change the production firewall on a guess. Copy SRE policy only onto a uniquely selected no-credential control Pod, alongside a separate deny-all control, allowing bounded propagation before TCP checks. Never modify the SRE policy or unrelated Pods; UID/RV-fence cleanup. Python: 80 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/common.py | 44 ++++++++++++++++++++++--- tests/e2e/sre_authority/harness_test.py | 27 ++++++++++++++- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index e6a479d71..b4b0e9fc8 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -572,6 +572,14 @@ def completed(): result["guardControls"][variant] = self.control_connectivity_diagnostics(pod, probe, variant) except Exception as error: result["guardControls"][variant] = {"diagnosticError": type(error).__name__} + try: + result["policyControl"] = self.policy_connectivity_diagnostics(pod, probe) + except Exception as error: + result["policyControl"] = {"diagnosticError": type(error).__name__} + try: + result["denyAllPolicyControl"] = self.policy_connectivity_diagnostics(pod, probe, deny_all=True) + except Exception as error: + result["denyAllPolicyControl"] = {"diagnosticError": type(error).__name__} return result @staticmethod @@ -586,17 +594,21 @@ def parse_connectivity_result(output): "Connectivity diagnostic produced an unexpected result") return result - def control_connectivity_diagnostics(self, pod, container, variant=None): + def control_connectivity_diagnostics(self, pod, container, variant=None, *, policy=False, deny_all=False): from .network_diagnostics import guard_variant - name = "sre-e2e-api-connectivity" + (f"-{variant}" if variant else "") + suffix = "-deny-all" if deny_all else "-policy" if policy else f"-{variant}" if variant else "" + name = "sre-e2e-api-connectivity" + suffix require(pod["spec"].get("nodeName"), "Connectivity comparison requires the actual sandbox node") spec = {"nodeName": pod["spec"]["nodeName"], "automountServiceAccountToken": False, "restartPolicy": "Never", "securityContext": copy.deepcopy(pod["spec"].get("securityContext", {})), "containers": [copy.deepcopy(container)]} if variant: spec["initContainers"] = [guard_variant(self.root, pod, variant)] - created = self.create({"apiVersion": "v1", "kind": "Pod", - "metadata": {"name": name, "namespace": TENANT}, "spec": spec}) + metadata = {"name": name, "namespace": TENANT} + if policy: + metadata["labels"] = {"kars.azure.com/e2e-policy-control": "deny-all" if deny_all else "true"} + spec["containers"][0]["command"][2] = "sleep 10\n" + spec["containers"][0]["command"][2] + created = self.create({"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}) uid = created["metadata"]["uid"] try: def completed(): @@ -606,7 +618,7 @@ def completed(): return status.get("phase") in ("Succeeded", "Failed") or any( item.get("state", {}).get("terminated", {}).get("exitCode", 0) != 0 for item in status.get("initContainerStatuses", [])) - self.poll("same-node credential-free TCP comparison", completed, seconds=25, interval=0.5) + self.poll("same-node credential-free TCP comparison", completed, seconds=35, interval=0.5) current = self.get("pod", name, TENANT) for item in current.get("status", {}).get("initContainerStatuses", []): code = item.get("state", {}).get("terminated", {}).get("exitCode", 0) @@ -625,6 +637,28 @@ def completed(): except Exception: print("SRE-DIAG owned connectivity control cleanup unavailable", flush=True) + def policy_connectivity_diagnostics(self, pod, container, *, deny_all=False): + source = self.get("networkpolicy", "sandbox-policy", RUNTIME) + require(source is not None, "SRE network policy is unavailable for a read-only comparison") + spec = {"policyTypes": ["Egress"], "egress": []} if deny_all else copy.deepcopy(source["spec"]) + spec["podSelector"] = {"matchLabels": {"kars.azure.com/e2e-policy-control": "deny-all" if deny_all else "true"}} + name = "sre-e2e-policy-control" + ("-deny-all" if deny_all else "") + created = self.create({"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": name, "namespace": TENANT}, "spec": spec}) + uid = created["metadata"]["uid"] + try: + return self.control_connectivity_diagnostics(pod, container, policy=True, deny_all=deny_all) + finally: + try: + current = self.get("networkpolicy", name, TENANT) + require(current and current["metadata"]["uid"] == uid, "Connectivity policy cleanup identity changed") + self.api("DELETE", f"/apis/networking.k8s.io/v1/namespaces/{TENANT}/networkpolicies/{name}", body={ + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, + status=(200, 202)) + except Exception: + print("SRE-DIAG owned connectivity policy cleanup unavailable", flush=True) + def close(self): for process in self.processes: process.terminate() diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index cd81aaad5..fc5acd378 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -67,10 +67,12 @@ def get(kind, _name, _namespace): h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) h.k = lambda *_args: '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}' h.control_connectivity_diagnostics = lambda *_args: {"available": False} + h.policy_connectivity_diagnostics = lambda *_args, **_kwargs: {"available": False} self.assertEqual(h.connectivity_diagnostics(pod), { "available": True, "uidMatches1001": True, "serviceExit": 0, "endpointExit": 0, "sameNodeControl": {"available": False}, - "guardControls": {variant: {"available": False} for variant in ("full", "filter-only", "legacy-full")}}) + "guardControls": {variant: {"available": False} for variant in ("full", "filter-only", "legacy-full")}, + "policyControl": {"available": False}, "denyAllPolicyControl": {"available": False}}) self.assertEqual(len(writes), 1) method, path, args = writes[0] self.assertEqual(method, "PATCH") @@ -120,6 +122,29 @@ def test_same_node_control_creates_only_unprivileged_owned_pod_and_uid_fenced_cl self.assertEqual(deleted[0][0], "DELETE") self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "control-uid", "resourceVersion": "7"}) + def test_policy_comparison_targets_only_owned_control_and_never_changes_source_policy(self): + h = Harness.__new__(Harness) + source = {"spec": {"podSelector": {}, "policyTypes": ["Egress"], + "egress": [{"to": [{"ipBlock": {"cidr": "10.96.0.1/32"}}], + "ports": [{"port": 443, "protocol": "TCP"}]}]}} + before = copy.deepcopy(source) + seen, deleted = [], [] + own = {"metadata": {"uid": "policy-uid", "resourceVersion": "3"}} + h.get = lambda kind, name, namespace: source if name == "sandbox-policy" else own + h.create = lambda obj: seen.append(obj) or own + h.control_connectivity_diagnostics = lambda *_args, **kwargs: {"policy": kwargs["policy"]} + h.api = lambda method, path, **kwargs: deleted.append((method, path, kwargs)) + self.assertEqual(h.policy_connectivity_diagnostics({}, {}), {"policy": True}) + self.assertEqual(source, before) + self.assertEqual(seen[0]["spec"]["egress"], source["spec"]["egress"]) + self.assertEqual(seen[0]["spec"]["podSelector"], + {"matchLabels": {"kars.azure.com/e2e-policy-control": "true"}}) + self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "policy-uid", "resourceVersion": "3"}) + h.policy_connectivity_diagnostics({}, {}, deny_all=True) + self.assertEqual(seen[1]["spec"], {"policyTypes": ["Egress"], "egress": [], + "podSelector": {"matchLabels": {"kars.azure.com/e2e-policy-control": "deny-all"}}}) + self.assertEqual(source, before) + def test_blocked_authority_diagnostics_report_source_coordinates_not_status_detail(self): root = Path(__file__).resolve().parents[3] cases = [ From 2700071c2a25d3c5219d9ce662e38a7208c776d6 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 20:22:11 +0200 Subject: [PATCH 41/62] test(e2e): prove exact post-DNAT API egress without agent bypass The copied SRE policy and separate deny-all policy both block the otherwise-working controls. In an owned empty control only, append exact ready API endpoint IP/HTTPS-port rules, retain the complete unchanged guard, and test actual UID 1001 connectivity alongside UID 1000 denial. No SRE production policy is changed. Preserve source policy bytes except the candidate rule/isolated selector and fence cleanup. Python: 81 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/common.py | 69 +++++++++++++++++++------ tests/e2e/sre_authority/harness_test.py | 42 ++++++++++++++- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index b4b0e9fc8..ff5506521 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -580,34 +580,51 @@ def completed(): result["denyAllPolicyControl"] = self.policy_connectivity_diagnostics(pod, probe, deny_all=True) except Exception as error: result["denyAllPolicyControl"] = {"diagnosticError": type(error).__name__} + try: + result["apiEndpointPolicyControl"] = self.policy_connectivity_diagnostics(pod, probe, api_endpoint=True) + except Exception as error: + result["apiEndpointPolicyControl"] = {"diagnosticError": type(error).__name__} return result @staticmethod - def parse_connectivity_result(output): + def parse_connectivity_result(output, uid=1001): + require(uid in (1000, 1001), "Unsupported connectivity probe UID") + uid_key = f"uidMatches{uid}" result = json.loads(output) require(isinstance(result, dict) and type(result.get("available")) is bool - and set(result) == ({"available", "uidMatches1001", "serviceExit", "endpointExit"} + and set(result) == ({"available", uid_key, "serviceExit", "endpointExit"} if result["available"] else {"available"}) - and (not result["available"] or type(result.get("uidMatches1001")) is bool) + and (not result["available"] or type(result.get(uid_key)) is bool) and all(type(value) is int and 0 <= value <= 255 - for key, value in result.items() if key not in ("available", "uidMatches1001")), + for key, value in result.items() if key not in ("available", uid_key)), "Connectivity diagnostic produced an unexpected result") return result - def control_connectivity_diagnostics(self, pod, container, variant=None, *, policy=False, deny_all=False): + def control_connectivity_diagnostics(self, pod, container, variant=None, *, policy=False, deny_all=False, api_endpoint=False): from .network_diagnostics import guard_variant - suffix = "-deny-all" if deny_all else "-policy" if policy else f"-{variant}" if variant else "" + suffix = "-endpoint" if api_endpoint else "-deny-all" if deny_all else "-policy" if policy else f"-{variant}" if variant else "" name = "sre-e2e-api-connectivity" + suffix require(pod["spec"].get("nodeName"), "Connectivity comparison requires the actual sandbox node") spec = {"nodeName": pod["spec"]["nodeName"], "automountServiceAccountToken": False, "restartPolicy": "Never", "securityContext": copy.deepcopy(pod["spec"].get("securityContext", {})), "containers": [copy.deepcopy(container)]} - if variant: - spec["initContainers"] = [guard_variant(self.root, pod, variant)] + if variant or api_endpoint: + spec["initContainers"] = [guard_variant(self.root, pod, variant or "full")] + if api_endpoint: + agent = copy.deepcopy(container) + agent["name"] = "agent-network" + agent["securityContext"]["runAsUser"] = 1000 + script = agent["command"][2] + require(script.count("= 1001") == 1 and script.count("uidMatches1001") == 1, + "Agent network comparison requires the exact diagnostic script") + agent["command"][2] = script.replace("= 1001", "= 1000").replace("uidMatches1001", "uidMatches1000") + spec["containers"].append(agent) metadata = {"name": name, "namespace": TENANT} if policy: - metadata["labels"] = {"kars.azure.com/e2e-policy-control": "deny-all" if deny_all else "true"} - spec["containers"][0]["command"][2] = "sleep 10\n" + spec["containers"][0]["command"][2] + value = "endpoint" if api_endpoint else "deny-all" if deny_all else "true" + metadata["labels"] = {"kars.azure.com/e2e-policy-control": value} + for probe in spec["containers"]: + probe["command"][2] = "sleep 10\n" + probe["command"][2] created = self.create({"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}) uid = created["metadata"]["uid"] try: @@ -624,8 +641,12 @@ def completed(): code = item.get("state", {}).get("terminated", {}).get("exitCode", 0) if code != 0: return {"available": False, "initExit": code} - return self.parse_connectivity_result( + result = self.parse_connectivity_result( self.k("logs", "-n", TENANT, name, "-c", container["name"], "--tail=5")) + if api_endpoint: + result["agentUid1000"] = self.parse_connectivity_result( + self.k("logs", "-n", TENANT, name, "-c", "agent-network", "--tail=5"), uid=1000) + return result finally: try: current = self.get("pod", name, TENANT) @@ -637,17 +658,35 @@ def completed(): except Exception: print("SRE-DIAG owned connectivity control cleanup unavailable", flush=True) - def policy_connectivity_diagnostics(self, pod, container, *, deny_all=False): + def policy_connectivity_diagnostics(self, pod, container, *, deny_all=False, api_endpoint=False): + import ipaddress source = self.get("networkpolicy", "sandbox-policy", RUNTIME) require(source is not None, "SRE network policy is unavailable for a read-only comparison") spec = {"policyTypes": ["Egress"], "egress": []} if deny_all else copy.deepcopy(source["spec"]) - spec["podSelector"] = {"matchLabels": {"kars.azure.com/e2e-policy-control": "deny-all" if deny_all else "true"}} - name = "sre-e2e-policy-control" + ("-deny-all" if deny_all else "") + if api_endpoint: + endpoint = self.get("endpoints", "kubernetes", "default") + rules = [] + for subset in endpoint.get("subsets", []): + for address in subset.get("addresses", []): + ip = ipaddress.ip_address(address["ip"]) + require(ip.is_private and not ip.is_loopback, "Unexpected diagnostic API endpoint address") + for port in subset.get("ports", []): + if port.get("name") == "https" and port.get("protocol", "TCP") == "TCP": + require(type(port["port"]) is int and 0 < port["port"] < 65536, + "Invalid diagnostic API endpoint port") + rules.append({"to": [{"ipBlock": {"cidr": f"{ip}/{ip.max_prefixlen}"}}], + "ports": [{"protocol": "TCP", "port": port["port"]}]}) + require(0 < len(rules) <= 16, "Diagnostic API endpoint inventory is empty or unbounded") + spec["egress"] = spec.get("egress", []) + rules + value = "endpoint" if api_endpoint else "deny-all" if deny_all else "true" + spec["podSelector"] = {"matchLabels": {"kars.azure.com/e2e-policy-control": value}} + name = "sre-e2e-policy-control" + ("-endpoint" if api_endpoint else "-deny-all" if deny_all else "") created = self.create({"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": {"name": name, "namespace": TENANT}, "spec": spec}) uid = created["metadata"]["uid"] try: - return self.control_connectivity_diagnostics(pod, container, policy=True, deny_all=deny_all) + return self.control_connectivity_diagnostics(pod, container, policy=True, + deny_all=deny_all, api_endpoint=api_endpoint) finally: try: current = self.get("networkpolicy", name, TENANT) diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index fc5acd378..071479bb7 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -72,7 +72,8 @@ def get(kind, _name, _namespace): "available": True, "uidMatches1001": True, "serviceExit": 0, "endpointExit": 0, "sameNodeControl": {"available": False}, "guardControls": {variant: {"available": False} for variant in ("full", "filter-only", "legacy-full")}, - "policyControl": {"available": False}, "denyAllPolicyControl": {"available": False}}) + "policyControl": {"available": False}, "denyAllPolicyControl": {"available": False}, + "apiEndpointPolicyControl": {"available": False}}) self.assertEqual(len(writes), 1) method, path, args = writes[0] self.assertEqual(method, "PATCH") @@ -122,6 +123,36 @@ def test_same_node_control_creates_only_unprivileged_owned_pod_and_uid_fenced_cl self.assertEqual(deleted[0][0], "DELETE") self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "control-uid", "resourceVersion": "7"}) + def test_endpoint_candidate_keeps_full_guard_and_tests_separate_agent_uid(self): + h = Harness.__new__(Harness) + h.root = Path(__file__).resolve().parents[3] + pod = {"spec": {"nodeName": "sandbox-worker"}} + container = {"name": "network", "image": "kars-sandbox-e2e:dev", + "command": ["sh", "-c", "uid test = 1001 uidMatches1001"], + "securityContext": {"runAsUser": 1001, "capabilities": {"drop": ["ALL"]}}} + seen = [] + current = {"metadata": {"uid": "control", "resourceVersion": "1"}, "status": {"phase": "Succeeded"}} + h.create = lambda obj: seen.append(obj) or current + h.get = lambda *_args: current + h.poll = lambda _label, predicate, **_kwargs: self.assertTrue(predicate()) + h.k = lambda *args: ( + '{"available":true,"uidMatches1000":true,"serviceExit":1,"endpointExit":124}' + if "agent-network" in args else '{"available":true,"uidMatches1001":true,"serviceExit":0,"endpointExit":0}') + h.api = lambda *_args, **_kwargs: None + with patch("sre_authority.network_diagnostics.guard_variant", return_value={"name": "guard"}) as guard: + result = h.control_connectivity_diagnostics(pod, container, policy=True, api_endpoint=True) + guard.assert_called_once_with(h.root, pod, "full") + actual = seen[0]["spec"]["containers"] + self.assertEqual([obj["securityContext"]["runAsUser"] for obj in actual], [1001, 1000]) + self.assertIn("= 1000", actual[1]["command"][2]) + self.assertIn("uidMatches1000", actual[1]["command"][2]) + self.assertTrue(actual[0]["command"][2].startswith("sleep 10\n")) + self.assertTrue(result["agentUid1000"]["uidMatches1000"]) + self.assertNotEqual(result["agentUid1000"]["endpointExit"], 0) + for obj in actual: + self.assertNotIn("volumeMounts", obj) + self.assertNotIn("envFrom", obj) + def test_policy_comparison_targets_only_owned_control_and_never_changes_source_policy(self): h = Harness.__new__(Harness) source = {"spec": {"podSelector": {}, "policyTypes": ["Egress"], @@ -136,12 +167,19 @@ def test_policy_comparison_targets_only_owned_control_and_never_changes_source_p h.api = lambda method, path, **kwargs: deleted.append((method, path, kwargs)) self.assertEqual(h.policy_connectivity_diagnostics({}, {}), {"policy": True}) self.assertEqual(source, before) + h.get = lambda kind, name, namespace: ( + {"subsets": [{"addresses": [{"ip": "172.18.0.2"}], "ports": [{"name": "https", "port": 6443}]}]} + if kind == "endpoints" else source if name == "sandbox-policy" else own) + h.policy_connectivity_diagnostics({}, {}, api_endpoint=True) + self.assertEqual(seen[1]["spec"]["egress"], source["spec"]["egress"] + [ + {"to": [{"ipBlock": {"cidr": "172.18.0.2/32"}}], "ports": [{"protocol": "TCP", "port": 6443}]}]) + self.assertEqual(source, before) self.assertEqual(seen[0]["spec"]["egress"], source["spec"]["egress"]) self.assertEqual(seen[0]["spec"]["podSelector"], {"matchLabels": {"kars.azure.com/e2e-policy-control": "true"}}) self.assertEqual(deleted[0][2]["body"]["preconditions"], {"uid": "policy-uid", "resourceVersion": "3"}) h.policy_connectivity_diagnostics({}, {}, deny_all=True) - self.assertEqual(seen[1]["spec"], {"policyTypes": ["Egress"], "egress": [], + self.assertEqual(seen[2]["spec"], {"policyTypes": ["Egress"], "egress": [], "podSelector": {"matchLabels": {"kars.azure.com/e2e-policy-control": "deny-all"}}}) self.assertEqual(source, before) From 6b01d03bd15b69443cf695ca40d9cbb1f4031578 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 20:44:47 +0200 Subject: [PATCH 42/62] fix(sre): allow exact ready API endpoints after service DNAT Real Kind policy controls prove that the Service VIP allowance misses the translated API endpoint. Adding only the exact endpoint IP/HTTPS port restores UID-1001 connectivity while UID 1000 remains blocked with the unchanged full guard. Discover only canonical live Service/Endpoints, validate the configured HTTPS target, bound/deduplicate ready same-family endpoint pairs, and fail closed on bad/incomplete data. Grant only named kubernetes Endpoints GET to the existing authority controller; refresh SRE reconciliation every 30s. Restore normal same-run CI build dependency and remove temporary verbose progress logging. Python 81 and Helm lint pass; all Rust compilation/tests remain hosted, no local Cargo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 16 +- controller/src/reconciler/mod.rs | 18 +- controller/src/reconciler/sre_egress.rs | 270 ++++++++++++++++++ .../kars/templates/sre-authority-rbac.yaml | 4 + docs/how-to/sre-authority.md | 9 + inference-router/src/sre_proxy/backend.rs | 17 -- inference-router/src/sre_proxy/mod.rs | 24 +- tests/e2e/sre_authority/binding_probe.py | 10 +- 8 files changed, 306 insertions(+), 62 deletions(-) create mode 100644 controller/src/reconciler/sre_egress.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a00abcb5..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -602,7 +602,7 @@ jobs: e2e-kind: name: E2E (Kind) - needs: [changes] + needs: [changes, build-rust] # Phase 3 S4: closes the audit gap "make test-e2e is not in CI". # Runs on every push to dev/main, manual dispatch, and PRs that # touch the runtime surface area (controller, router, helm chart, @@ -702,20 +702,8 @@ jobs: if: steps.paths.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kars-binaries-2945756ef265d41d984b8a4b1615b6a1a1291d53 - run-id: 34370014115 - github-token: ${{ github.token }} + name: kars-binaries-${{ github.sha }} path: ./bin/ - - name: Verify diagnostic binary production equivalence and checksums - if: steps.paths.outputs.run == 'true' - run: | - git fetch --no-tags --depth=1 origin 2945756ef265d41d984b8a4b1615b6a1a1291d53 - changed=$(git diff --name-only 2945756ef265d41d984b8a4b1615b6a1a1291d53 HEAD) - if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|\.github/workflows/ci\.yml$|$)'; then - echo "Refusing diagnostic binaries from different production sources" - exit 1 - fi - (cd bin && sha256sum --check SHA256SUMS) - name: chmod binaries if: steps.paths.outputs.run == 'true' # Per-arch layout (./bin/amd64/) matches release-internal.yml's diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 75d1b7d66..3cf68b9ca 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -47,6 +47,7 @@ pub(crate) mod trustgraph_mount; use mcp_egress::mcp_egress_rule; mod pod_spec; +mod sre_egress; pub(crate) use pod_spec::{ build_egress_guard_command, build_pod_security_context, isolation_scheduling, sandbox_node_selector_from, @@ -937,13 +938,14 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result().unwrap_or(443)}] - })); + // Policy enforcement may observe the API's post-DNAT endpoint IP/port. + // Only registered routers receive exact targets; UID 1000 stays locked down. + if sre_projection.is_some() { + egress_rules.extend( + sre_egress::rules(client, &apiserver_ip, &apiserver_port) + .await + .map_err(ReconcileError::Configuration)?, + ); } // Add user-defined allowed endpoints (for the inference-router to reach @@ -3104,7 +3106,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result String { + match error { + kube::Error::Api(response) => { + format!( + "Read canonical API {resource}: Kubernetes status {}", + response.code + ) + } + _ => format!("Read canonical API {resource}: Kubernetes transport failure"), + } +} + +pub(super) async fn rules( + client: &Client, + service_host: &str, + service_port: &str, +) -> Result, String> { + let service = Api::::namespaced(client.clone(), "default") + .get("kubernetes") + .await + .map_err(|error| read_error("Service", error))?; + let endpoints = Api::::namespaced(client.clone(), "default") + .get("kubernetes") + .await + .map_err(|error| read_error("Endpoints", error))?; + from_objects(&service, &endpoints, service_host, service_port) +} + +fn canonical(metadata: &ObjectMeta) -> bool { + metadata.name.as_deref() == Some("kubernetes") + && metadata.namespace.as_deref() == Some("default") + && metadata + .uid + .as_deref() + .is_some_and(|value| !value.is_empty()) + && metadata + .resource_version + .as_deref() + .is_some_and(|value| !value.is_empty()) + && metadata.deletion_timestamp.is_none() +} + +fn address(value: &str) -> Result { + let ip: IpAddr = value + .parse() + .map_err(|_| "Invalid API endpoint IP address")?; + if ip.is_unspecified() + || ip.is_loopback() + || ip.is_multicast() + || matches!(ip, IpAddr::V4(ip) if ip.is_broadcast() || ip.is_link_local()) + || matches!(ip, IpAddr::V6(ip) if ip.is_unicast_link_local()) + { + return Err("API endpoint must be a routable unicast IP".into()); + } + Ok(ip) +} + +fn port(value: i32) -> Result { + u16::try_from(value) + .ok() + .filter(|value| *value != 0) + .ok_or_else(|| "Invalid API HTTPS port".into()) +} + +fn rule(ip: IpAddr, port: u16) -> Value { + let prefix = if ip.is_ipv4() { 32 } else { 128 }; + json!({ + "to": [{"ipBlock": {"cidr": format!("{ip}/{prefix}")}}], + "ports": [{"protocol": "TCP", "port": port}], + }) +} + +fn from_objects( + service: &Service, + endpoints: &Endpoints, + service_host: &str, + service_port: &str, +) -> Result, String> { + if !canonical(&service.metadata) || !canonical(&endpoints.metadata) { + return Err("Canonical API Service/Endpoints identity is missing or terminating".into()); + } + let host = address(service_host)?; + let configured_port = port( + service_port + .parse() + .map_err(|_| "Invalid configured API HTTPS port")?, + )?; + let spec = service + .spec + .as_ref() + .ok_or("Canonical API Service spec is missing")?; + if spec.cluster_ip.as_deref().map(address).transpose()? != Some(host) + || !spec + .ports + .as_deref() + .unwrap_or_default() + .iter() + .any(|entry| { + entry.name.as_deref() == Some("https") + && entry.protocol.as_deref().unwrap_or("TCP") == "TCP" + && entry.port == i32::from(configured_port) + }) + { + return Err("Canonical API Service does not match the controller's HTTPS target".into()); + } + let mut targets = BTreeSet::new(); + for subset in endpoints.subsets.as_deref().unwrap_or_default() { + for entry in subset.ports.as_deref().unwrap_or_default() { + if entry.name.as_deref() != Some("https") { + continue; + } + if entry.protocol.as_deref().unwrap_or("TCP") != "TCP" { + return Err("API HTTPS endpoint must use TCP".into()); + } + let endpoint_port = port(entry.port)?; + // Only ready addresses are eligible; never include notReadyAddresses. + for entry in subset.addresses.as_deref().unwrap_or_default() { + let ip = address(&entry.ip)?; + if ip.is_ipv4() != host.is_ipv4() { + continue; + } + targets.insert((ip, endpoint_port)); + if targets.len() > MAX_ENDPOINTS { + return Err("Canonical API endpoint inventory exceeds its bounded limit".into()); + } + } + } + } + if targets.is_empty() { + return Err("Canonical API Service has no ready HTTPS endpoints".into()); + } + let mut rules = vec![rule(host, configured_port)]; + rules.extend( + targets + .into_iter() + .filter(|target| *target != (host, configured_port)) + .map(|(ip, port)| rule(ip, port)), + ); + Ok(rules) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixtures() -> (Service, Endpoints) { + let metadata = + json!({"name":"kubernetes","namespace":"default","uid":"live","resourceVersion":"1"}); + ( + serde_json::from_value(json!({"metadata":metadata,"spec":{ + "clusterIP":"10.96.0.1","ports":[{"name":"https","port":443,"protocol":"TCP"}]}})) + .unwrap(), + serde_json::from_value(json!({"metadata":metadata,"subsets":[{ + "addresses":[{"ip":"172.18.0.2"},{"ip":"172.18.0.2"}], + "notReadyAddresses":[{"ip":"172.18.0.99"}], + "ports":[{"name":"https","port":6443,"protocol":"TCP"}]}]})) + .unwrap(), + ) + } + + #[test] + fn exact_service_and_post_dnat_targets_are_paired_deduplicated_and_ready_only() { + let (service, endpoints) = fixtures(); + assert_eq!( + from_objects(&service, &endpoints, "10.96.0.1", "443").unwrap(), + vec![ + rule("10.96.0.1".parse().unwrap(), 443), + rule("172.18.0.2".parse().unwrap(), 6443) + ] + ); + } + + #[test] + fn ipv6_uses_only_exact_ready_targets_of_the_service_family() { + let (mut service, mut endpoints) = fixtures(); + service.spec.as_mut().unwrap().cluster_ip = Some("fd00::1".into()); + endpoints.subsets.as_mut().unwrap()[0] + .addresses + .as_mut() + .unwrap()[0] + .ip = "fd01::2".into(); + assert_eq!( + from_objects(&service, &endpoints, "fd00::1", "443").unwrap(), + vec![ + rule("fd00::1".parse().unwrap(), 443), + rule("fd01::2".parse().unwrap(), 6443) + ] + ); + } + + #[test] + fn missing_identity_mismatched_service_and_unready_inventory_fail_closed() { + let (service, endpoints) = fixtures(); + assert!(from_objects(&service, &endpoints, "10.96.0.2", "443").is_err()); + assert!(from_objects(&service, &endpoints, "10.96.0.1", "444").is_err()); + assert!(from_objects(&service, &endpoints, "10.96.0.1", "invalid").is_err()); + let mut invalid = service.clone(); + invalid.metadata.namespace = Some("other".into()); + assert!(from_objects(&invalid, &endpoints, "10.96.0.1", "443").is_err()); + let mut invalid = endpoints.clone(); + invalid.metadata.uid = None; + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + let mut invalid = endpoints.clone(); + invalid.subsets.as_mut().unwrap()[0].addresses = None; + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + + #[test] + fn malformed_addresses_ports_or_protocols_never_produce_partial_rules() { + let (service, endpoints) = fixtures(); + for ip in [ + "invalid", + "0.0.0.0", + "127.0.0.1", + "169.254.169.254", + "224.0.0.1", + "255.255.255.255", + ] { + let mut invalid = endpoints.clone(); + invalid.subsets.as_mut().unwrap()[0] + .addresses + .as_mut() + .unwrap()[0] + .ip = ip.into(); + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + for value in [0, -1, 65536] { + let mut invalid = endpoints.clone(); + invalid.subsets.as_mut().unwrap()[0].ports.as_mut().unwrap()[0].port = value; + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + let mut invalid = endpoints; + invalid.subsets.as_mut().unwrap()[0].ports.as_mut().unwrap()[0].protocol = + Some("UDP".into()); + assert!(from_objects(&service, &invalid, "10.96.0.1", "443").is_err()); + } + + #[test] + fn endpoint_expansion_is_bounded_and_read_errors_never_echo_api_bodies() { + let (service, mut endpoints) = fixtures(); + endpoints.subsets.as_mut().unwrap()[0].addresses = Some( + (1..=33) + .map(|index| { + serde_json::from_value(json!({"ip":format!("172.18.1.{index}")})).unwrap() + }) + .collect(), + ); + assert!(from_objects(&service, &endpoints, "10.96.0.1", "443").is_err()); + let error = kube::Error::Api(serde_json::from_value(json!({ + "status":"Failure","message":"private-response-must-not-log","reason":"Forbidden","code":403 + })).unwrap()); + assert_eq!( + read_error("Endpoints", error), + "Read canonical API Endpoints: Kubernetes status 403" + ); + } +} diff --git a/deploy/helm/kars/templates/sre-authority-rbac.yaml b/deploy/helm/kars/templates/sre-authority-rbac.yaml index 819aef1c2..c3580fb6d 100644 --- a/deploy/helm/kars/templates/sre-authority-rbac.yaml +++ b/deploy/helm/kars/templates/sre-authority-rbac.yaml @@ -117,6 +117,10 @@ rules: - apiGroups: ["authorization.k8s.io"] resources: ["subjectaccessreviews"] verbs: ["create"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["kubernetes"] + verbs: ["get"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"] verbs: ["get", "list", "watch"] diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index dcaa6da4c..c4f10432d 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -214,6 +214,15 @@ and requires cluster-wide `apps/replicasets` CREATE authority; namespaced workload permissions are insufficient. It does not grant the Deployment controller Pod CREATE or registrar authority. +The registered router's API egress includes the canonical Service IP and its +ready HTTPS endpoint IP/port pairs, since policy enforcement can see the +post-DNAT destination rather than the Service VIP. These are exact `/32` or +`/128` targets, not private-subnet allowances; malformed, missing, terminating, +or oversized endpoint inventories fail closed. The controller fetches +`default/kubernetes`, with only named `kubernetes` Endpoints GET added to its +authority role, and refreshes registered SRE reconciliation every 30 seconds. +The UID-1000 egress guard and opaque agent credential are unchanged. + The proxy checks current registration and live UID/claim authority. It permits the bounded first-party diagnostic read/log/metrics paths and Pending-only `KarsSREAction` creation in `kars-sre`. Secret responses retain key names but diff --git a/inference-router/src/sre_proxy/backend.rs b/inference-router/src/sre_proxy/backend.rs index 71b98230e..c59ba510a 100644 --- a/inference-router/src/sre_proxy/backend.rs +++ b/inference-router/src/sre_proxy/backend.rs @@ -54,15 +54,6 @@ fn denied_response(stage: &'static str, status: reqwest::StatusCode) { ); } -fn authority_progress(stage: &'static str, step: &'static str) { - tracing::debug!( - target: "inference_router::sre_proxy", - stage, - step, - "SRE authority progress" - ); -} - pub(super) struct Backend { pub config: Config, client: reqwest::Client, @@ -207,9 +198,7 @@ impl Backend { } async fn metadata_json(&self, path: &str, stage: &'static str) -> Result { - authority_progress(stage, "token"); let token = self.bearer().await?; - authority_progress(stage, "request"); let response = self .client .get(format!( @@ -225,7 +214,6 @@ impl Backend { transport_failure(stage, &error); "SRE authority read failed" })?; - authority_progress(stage, "headers"); if !response.status().is_success() { denied_response(stage, response.status()); return Err("SRE authority read denied".into()); @@ -234,7 +222,6 @@ impl Backend { .json() .await .map_err(|_| "SRE authority response invalid".to_string())?; - authority_progress(stage, "complete"); Ok(value) } @@ -314,7 +301,6 @@ impl Backend { } async fn verify_privacy(&self) -> Result<(), String> { - authority_progress("privacy-review", "request"); for review in crate::sre_privacy::secret_access_reviews(&self.config.runtime_namespace) { let response = self .client @@ -340,8 +326,6 @@ impl Backend { .map_err(|_| "SRE privacy authorization response invalid")?; crate::sre_privacy::require_denial(&response)?; } - authority_progress("privacy-review", "complete"); - authority_progress("credential-metadata", "request"); let response = self .client .get(format!( @@ -372,7 +356,6 @@ impl Backend { &metadata, &[self.config.service_account_uid.as_str()], )?; - authority_progress("credential-metadata", "complete"); Ok(()) } diff --git a/inference-router/src/sre_proxy/mod.rs b/inference-router/src/sre_proxy/mod.rs index 247c7ff0d..55160ad82 100644 --- a/inference-router/src/sre_proxy/mod.rs +++ b/inference-router/src/sre_proxy/mod.rs @@ -76,16 +76,7 @@ fn readiness_failure_category(reason: &str) -> &'static str { } } -fn transport_progress(stage: &'static str) { - tracing::debug!( - target: "inference_router::sre_proxy", - stage, - "SRE transport progress" - ); -} - async fn ready(State(proxy): State) -> Response { - transport_progress("readiness-entered"); let Ok(_permit) = proxy.capacity.try_acquire() else { return error( StatusCode::TOO_MANY_REQUESTS, @@ -93,7 +84,6 @@ async fn ready(State(proxy): State) -> Response { ); }; let started = std::time::Instant::now(); - transport_progress("authority-entered"); let authorization = proxy.backend.authorize().await; let elapsed_seconds = started.elapsed().as_secs(); if elapsed_seconds >= 2 { @@ -278,19 +268,13 @@ impl axum::serve::Listener for Listener { loop { match self.tcp.accept().await { Ok((stream, address)) => { - transport_progress("tcp-accepted"); - match tokio::time::timeout( + if let Ok(Ok(stream)) = tokio::time::timeout( std::time::Duration::from_secs(3), self.tls.accept(stream), ) .await { - Ok(Ok(stream)) => { - transport_progress("tls-accepted"); - return (stream, address); - } - Ok(Err(_)) => transport_progress("tls-rejected"), - Err(_) => transport_progress("tls-timeout"), + return (stream, address); } } Err(_) => tokio::time::sleep(std::time::Duration::from_millis(100)).await, @@ -324,7 +308,6 @@ pub async fn start() -> Result>, String> { if std::env::var("KARS_SRE_API_ENABLED").as_deref() != Ok("true") { return Ok(None); } - transport_progress("enabled"); let directory = PathBuf::from(DIRECTORY); let backend = Backend::load(&directory)?; let token = std::fs::read_to_string(directory.join("agent-token")) @@ -332,14 +315,12 @@ pub async fn start() -> Result>, String> { if token.trim().len() != 64 || !token.trim().bytes().all(|b| b.is_ascii_alphanumeric()) { return Err("SRE proxy credential invalid".into()); } - transport_progress("loaded"); let listener = Listener { tcp: TcpListener::bind(("127.0.0.1", PORT)) .await .map_err(|_| "SRE loopback TLS listener unavailable")?, tls: tls(&directory)?, }; - transport_progress("bound"); backend.renew_in_background(); let proxy = Proxy { backend, @@ -348,7 +329,6 @@ pub async fn start() -> Result>, String> { }; let router = app(proxy); Ok(Some(tokio::spawn(async move { - transport_progress("serving"); if axum::serve(listener, router).await.is_err() { tracing::error!("SRE TLS proxy stopped; router must restart"); std::process::exit(1); diff --git a/tests/e2e/sre_authority/binding_probe.py b/tests/e2e/sre_authority/binding_probe.py index 203f60df3..0ece4efcc 100644 --- a/tests/e2e/sre_authority/binding_probe.py +++ b/tests/e2e/sre_authority/binding_probe.py @@ -249,7 +249,15 @@ def prove(root, port, state, objects, report): facts["authorization"] = {"patchReaderBinding": True, "useCanonical": True, "bindReader": baseline_bind, "bindCustom": api.allowed("rbac.authorization.k8s.io", "clusterroles", "bind", CUSTOM), "getCustomLimitRanges": api.allowed("", "limitranges", "get"), - "getReaderNodeMetrics": api.allowed("metrics.k8s.io", "nodes", "get")} + "getReaderNodeMetrics": api.allowed("metrics.k8s.io", "nodes", "get"), + "getApiEndpoints": api.allowed("", "endpoints", "get", "kubernetes"), + "getOtherEndpoints": api.allowed("", "endpoints", "get", "unrelated"), + "listEndpoints": api.allowed("", "endpoints", "list")} + report(facts) + require(facts["authorization"]["getApiEndpoints"] + and not facts["authorization"]["getOtherEndpoints"] + and not facts["authorization"]["listEndpoints"], + "API endpoint discovery is missing or broader than the exact named GET") require(baseline_bind, "Shipped controller lacks the exact named legacy reader bind permission") facts["shippedReaderBindVerified"] = True # The historical 403/200/403 experiment is retained in its immutable CI From 42e2fcd7f39ae449aa3eaa676bc8ef61cef4766b Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 21:30:06 +0200 Subject: [PATCH 43/62] test(e2e): expose safe HTTP status at the real Hermes boundary The post-DNAT correction passes actual install and router readiness; the next unchanged-Hermes Secret GET raises HTTPStatusError. Report only HTTP status, allowed Status reason, checked-in response-source coordinates and harness call site; never exception text/URLs/headers/body data. Collect bounded authority facts from Ready routers too. Temporarily replay full Kind with verified source-equivalent 6b01d03b CI binaries while Rust gates run independently. Python: 82 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 16 +++++++- tests/e2e/sre-authority.py | 6 +++ .../sre_authority/bootstrap_diagnostics.py | 37 +++++++++++++++++++ .../e2e/sre_authority/bootstrap_probe_test.py | 19 +++++++++- tests/e2e/sre_authority/common.py | 9 +++++ 5 files changed, 84 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7157bc56c..d2d577b51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -602,7 +602,7 @@ jobs: e2e-kind: name: E2E (Kind) - needs: [changes, build-rust] + needs: [changes] # Phase 3 S4: closes the audit gap "make test-e2e is not in CI". # Runs on every push to dev/main, manual dispatch, and PRs that # touch the runtime surface area (controller, router, helm chart, @@ -702,8 +702,20 @@ jobs: if: steps.paths.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kars-binaries-${{ github.sha }} + name: kars-binaries-2b93664e3b7630fd302a4b4f1c3df232e6752eb5 + run-id: 34390909456 + github-token: ${{ github.token }} path: ./bin/ + - name: Verify diagnostic binary production equivalence and checksums + if: steps.paths.outputs.run == 'true' + run: | + git fetch --no-tags --depth=1 origin 2b93664e3b7630fd302a4b4f1c3df232e6752eb5 + changed=$(git diff --name-only 2b93664e3b7630fd302a4b4f1c3df232e6752eb5 HEAD) + if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|tests/e2e/sre-authority\.py$|\.github/workflows/ci\.yml$|$)'; then + echo "Refusing diagnostic binaries from different production sources" + exit 1 + fi + (cd bin && sha256sum --check SHA256SUMS) - name: chmod binaries if: steps.paths.outputs.run == 'true' # Per-arch layout (./bin/amd64/) matches release-internal.yml's diff --git a/tests/e2e/sre-authority.py b/tests/e2e/sre-authority.py index 0b0edead5..8a68639d1 100644 --- a/tests/e2e/sre-authority.py +++ b/tests/e2e/sre-authority.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. import argparse +import json from pathlib import Path @@ -54,6 +55,11 @@ def main(): # command output, JWTs and TLS private keys never become failure logs. print(f"SRE-FAIL {options.phase}: {str(error) if isinstance(error, AssertionError) else type(error).__name__}", flush=True) if harness: + from sre_authority.bootstrap_diagnostics import exception_summary + try: + print("SRE-ERROR", json.dumps(exception_summary(error, harness.root)), flush=True) + except Exception: + print("SRE-ERROR sanitized exception summary unavailable", flush=True) harness.diagnostics() return 1 finally: diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 7104b1634..17ab949a3 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -154,6 +154,43 @@ def firewall_summary(text): return result +def exception_summary(error, root): + result = {"category": type(error).__name__} + frame = error.__traceback__ + while frame: + source = str(frame.tb_frame.f_code.co_filename) + prefix = str(root / "tests/e2e/sre_authority") + "/" + if source.startswith(prefix): + result["callSite"] = {"source": source[len(str(root)) + 1:], + "function": frame.tb_frame.f_code.co_name, "line": frame.tb_lineno} + frame = frame.tb_next + response = getattr(error, "response", None) + status = getattr(response, "status_code", None) + if type(status) is not int or not 100 <= status <= 599: + return result + result["httpStatus"] = status + try: + body = response.json() + except (ValueError, TypeError): + return result + if not isinstance(body, dict) or body.get("kind") != "Status": + return result + reason = body.get("reason") + if isinstance(reason, str) and (reason in REASONS or reason == "SREProxyDenied"): + result["reason"] = reason + message = body.get("message") + if not isinstance(message, str) or not 0 < len(message) <= 4096: + return result + literal = json.dumps(message, ensure_ascii=False) + for name in ("mod.rs", "policy.rs", "backend.rs"): + path = root / "inference-router/src/sre_proxy" / name + for number, source in enumerate(path.read_text().splitlines(), 1): + if literal in source: + result["responseSite"] = {"source": str(path.relative_to(root)), "line": number} + return result + return result + + def identifier(value): return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:/-]{1,253}", value) else None diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 3bbcd33f5..e0d2b01b7 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -7,7 +7,7 @@ import unittest from unittest.mock import patch -from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, failure_facts, firewall_summary, object_status, policy_status, probe_command_result, public_stack_facts, router_log_summary, router_readiness_facts +from sre_authority.bootstrap_diagnostics import api_result, collect, control_plane_status, exception_summary, failure_facts, firewall_summary, object_status, policy_status, probe_command_result, public_stack_facts, router_log_summary, router_readiness_facts from sre_authority.bootstrap_probe import builtin_documents, converted_objects, exercise, preserved_json_candidate, safe_controller POLICIES = {"kars-sre-private-mounts": {"spec": {"validations": [ @@ -15,6 +15,23 @@ class BootstrapProofTests(unittest.TestCase): + def test_http_failure_summary_reports_status_and_checked_source_not_body_url_or_headers(self): + root = Path(__file__).resolve().parents[3] + class Response: + status_code = 401 + def json(self): + return {"kind": "Status", "reason": "SREProxyDenied", + "message": "An SRE proxy credential is required", "data": "do-not-publish"} + error = RuntimeError("do-not-publish URL/header/credential") + error.response = Response() + facts = exception_summary(error, root) + self.assertEqual(facts["httpStatus"], 401) + self.assertTrue(facts["responseSite"]["source"].endswith("sre_proxy/mod.rs")) + self.assertNotIn("do-not-publish", json.dumps(facts)) + error.response.json = lambda: {"kind": "Status", "message": "do-not-publish"} + self.assertNotIn("responseSite", exception_summary(error, root)) + self.assertNotIn("do-not-publish", json.dumps(exception_summary(error, root))) + def test_firewall_summary_does_not_publish_rules_comments_addresses_or_unknown_chains(self): text = """*filter :OUTPUT ACCEPT [12:640] diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index ff5506521..eec1cfae7 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -413,6 +413,15 @@ def diagnostics(self): if container["name"] == "inference-router"), None) if router and not router.get("ready") and "running" in router.get("state", {}): self.readiness_diagnostics(pod) + elif router and router.get("ready"): + from .bootstrap_diagnostics import router_readiness_facts + try: + logs = self.k("logs", "-n", RUNTIME, pod["metadata"]["name"], "-c", "inference-router", + "--tail=150", timeout=10) + print("SRE-DIAG", json.dumps({"kind": "RouterAuthorityLog", "podUid": pod["metadata"]["uid"], + "authorityChecks": router_readiness_facts(logs)}), flush=True) + except Exception: + print("SRE-DIAG Ready router authority log unavailable", flush=True) except Exception: print("SRE-DIAG runtime Pod status unavailable", flush=True) From 026cda4fc75bfb5f342a40b6ec2786f1280fd029 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 21:48:10 +0200 Subject: [PATCH 44/62] fix(sre): filter native SecretList items without per-item TypeMeta Actual unchanged-Hermes GET succeeds; LIST returns 502 from the per-item Secret validator. Accept omitted kind/apiVersion only within a typed SecretList envelope, reject conflicting explicit types and malformed metadata, and retain the exact metadata whitelist/value erasure. Exercise native typeless list items in the HTTPS integration fixture and record only owned synthetic-Secret wire-shape booleans in real Kind. Restore normal same-run CI binaries; Python 83 passed, no local Cargo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .github/workflows/ci.yml | 16 +------ docs/how-to/sre-authority.md | 2 + inference-router/src/sre_proxy/policy.rs | 59 ++++++++++++++++++++++-- inference-router/src/sre_proxy/tests.rs | 7 ++- tests/e2e/sre_authority/harness_test.py | 14 +++++- tests/e2e/sre_authority/proxy.py | 16 ++++++- 6 files changed, 92 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2d577b51..7157bc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -602,7 +602,7 @@ jobs: e2e-kind: name: E2E (Kind) - needs: [changes] + needs: [changes, build-rust] # Phase 3 S4: closes the audit gap "make test-e2e is not in CI". # Runs on every push to dev/main, manual dispatch, and PRs that # touch the runtime surface area (controller, router, helm chart, @@ -702,20 +702,8 @@ jobs: if: steps.paths.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kars-binaries-2b93664e3b7630fd302a4b4f1c3df232e6752eb5 - run-id: 34390909456 - github-token: ${{ github.token }} + name: kars-binaries-${{ github.sha }} path: ./bin/ - - name: Verify diagnostic binary production equivalence and checksums - if: steps.paths.outputs.run == 'true' - run: | - git fetch --no-tags --depth=1 origin 2b93664e3b7630fd302a4b4f1c3df232e6752eb5 - changed=$(git diff --name-only 2b93664e3b7630fd302a4b4f1c3df232e6752eb5 HEAD) - if printf '%s\n' "$changed" | grep -vE '^(tests/e2e/sre_authority/|tests/e2e/sre-authority\.py$|\.github/workflows/ci\.yml$|$)'; then - echo "Refusing diagnostic binaries from different production sources" - exit 1 - fi - (cd bin && sha256sum --check SHA256SUMS) - name: chmod binaries if: steps.paths.outputs.run == 'true' # Per-arch layout (./bin/amd64/) matches release-internal.yml's diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index c4f10432d..1f296179f 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -227,6 +227,8 @@ The proxy checks current registration and live UID/claim authority. It permits the bounded first-party diagnostic read/log/metrics paths and Pending-only `KarsSREAction` creation in `kars-sre`. Secret responses retain key names but exclude values, `stringData`, annotations, labels, and managed-field copies. +The typed `SecretList` envelope supplies item types when Kubernetes omits item +`kind`/`apiVersion`; conflicting explicit type metadata is still rejected. Encoded/noncanonical paths, watches, streaming log follow, token requests, exec/proxy subresources, arbitrary writes, and non-JSON media escapes are denied. The existing Hermes proposal builder's two diagnostic labels are accepted; diff --git a/inference-router/src/sre_proxy/policy.rs b/inference-router/src/sre_proxy/policy.rs index 73b2a8c07..58d1d76ec 100644 --- a/inference-router/src/sre_proxy/policy.rs +++ b/inference-router/src/sre_proxy/policy.rs @@ -180,8 +180,14 @@ fn validate_query(query: Option<&str>, route: Route) -> Result<(), &'static str> Ok(()) } -fn secret(value: &Value) -> Result { - if value["kind"] != "Secret" || !value["metadata"].is_object() { +fn secret(value: &Value, list_item: bool) -> Result { + let kind_matches = value.get("kind").map_or(list_item, |kind| kind == "Secret"); + if !kind_matches + || !value["metadata"].is_object() + || value + .get("apiVersion") + .is_some_and(|version| version != "v1") + { return Err("Malformed Secret response"); } let mut metadata = serde_json::Map::new(); @@ -212,13 +218,21 @@ fn secret(value: &Value) -> Result { pub(super) fn secret_projection(value: &Value) -> Result { if value["kind"] == "Secret" { - return secret(value); + return secret(value, false); } - if value["kind"] != "SecretList" { + if value["kind"] != "SecretList" + || value + .get("apiVersion") + .is_some_and(|version| version != "v1") + { return Err("Unexpected Secret response kind"); } let items = value["items"].as_array().ok_or("Malformed Secret list")?; - let items = items.iter().map(secret).collect::, _>>()?; + // Kubernetes omits TypeMeta on items inside its typed list envelope. + let items = items + .iter() + .map(|item| secret(item, true)) + .collect::, _>>()?; Ok(json!({"apiVersion":"v1","kind":"SecretList", "metadata":{"resourceVersion":value["metadata"]["resourceVersion"],"continue":value["metadata"]["continue"]}, "items":items})) @@ -378,6 +392,41 @@ mod tests { } } + #[test] + fn native_secret_list_items_may_omit_typemeta_but_not_conflict_with_the_envelope() { + let item = json!({"metadata":{"name":"test","namespace":"kars-test","uid":"uid","resourceVersion":"7", + "annotations":{"copy":"PRIVATE_VALUE"},"labels":{"copy":"PRIVATE_VALUE"}}, + "type":"Opaque","data":{"key":"PRIVATE_VALUE"},"stringData":{"copy":"PRIVATE_VALUE"}}); + let list = json!({"apiVersion":"v1","kind":"SecretList", + "metadata":{"resourceVersion":"9","continue":"cursor"},"items":[item.clone()]}); + let output = secret_projection(&list).unwrap(); + assert_eq!(output["items"][0]["kind"], "Secret"); + assert_eq!(output["items"][0]["apiVersion"], "v1"); + assert_eq!(output["items"][0]["data"], json!({"key":""})); + assert_eq!(output["metadata"], list["metadata"]); + assert!(!output.to_string().contains("PRIVATE_VALUE")); + assert!( + !output["items"][0]["metadata"] + .as_object() + .unwrap() + .contains_key("annotations") + ); + assert!(secret_projection(&item).is_err()); + for (key, value) in [ + ("kind", json!("ConfigMap")), + ("kind", Value::Null), + ("apiVersion", json!("other/v1")), + ("metadata", Value::Null), + ] { + let mut invalid = list.clone(); + invalid["items"][0][key] = value; + assert!(secret_projection(&invalid).is_err()); + } + let mut invalid = list; + invalid["kind"] = "List".into(); + assert!(secret_projection(&invalid).is_err()); + } + #[test] fn proposals_cannot_self_approve_or_inject_status_ownership_or_extra_fields() { let base = json!({"apiVersion":"kars.azure.com/v1alpha1","kind":"KarsSREAction", diff --git a/inference-router/src/sre_proxy/tests.rs b/inference-router/src/sre_proxy/tests.rs index 5ebd30586..1ec0ebf47 100644 --- a/inference-router/src/sre_proxy/tests.rs +++ b/inference-router/src/sre_proxy/tests.rs @@ -93,7 +93,12 @@ async fn fixture() -> Fixture { json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{},"items":state.aliases}) } "/api/v1/namespaces/kars-demo/secrets/router-services-admin" => secret(), - "/api/v1/secrets" => json!({"apiVersion":"v1","kind":"SecretList","metadata":{},"items":[secret()]}), + "/api/v1/secrets" => { + let mut item = secret(); + item.as_object_mut().unwrap().remove("kind"); + item.as_object_mut().unwrap().remove("apiVersion"); + json!({"apiVersion":"v1","kind":"SecretList","metadata":{},"items":[item]}) + } "/api/v1/namespaces/kars-demo/pods/app/log" => return ResponseTemplate::new(200).set_body_raw("legitimate pod log\n","text/plain"), "/apis/metrics.k8s.io/v1beta1/nodes" => json!({"kind":"NodeMetricsList","items":[]}), "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions" if request.method=="POST" => { diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 071479bb7..bda59356a 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -20,7 +20,7 @@ ) from sre_authority.fixtures import seed_control_consumer from sre_authority.admission import reserved_source_probe -from sre_authority.proxy import MARKER, assert_filtered +from sre_authority.proxy import MARKER, assert_filtered, secret_list_wire_facts from sre_authority.credential_paths import assert_token_type_immutable, assert_watch_result @@ -34,6 +34,18 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_native_secret_list_wire_facts_publish_no_synthetic_secret_fields(self): + value = {"kind": "SecretList", "items": [{"metadata": {"uid": "fixture", "annotations": {"copy": MARKER}}, + "data": {"token": MARKER}}]} + facts = secret_list_wire_facts(value, "fixture") + self.assertEqual(facts, {"envelopeKind": "SecretList", "itemHasKind": False, + "itemHasApiVersion": False, "itemMetadataObject": True}) + self.assertNotIn(MARKER, json.dumps(facts)) + with self.assertRaises(AssertionError): + secret_list_wire_facts(value, "other") + with self.assertRaises(AssertionError): + secret_list_wire_facts({"kind": "List", "items": []}, "fixture") + def test_firewall_diagnostic_requires_owned_node_before_readonly_namespace_entry(self): h = Harness.__new__(Harness) calls = [] diff --git a/tests/e2e/sre_authority/proxy.py b/tests/e2e/sre_authority/proxy.py index f39eb6fca..0730ea8ae 100644 --- a/tests/e2e/sre_authority/proxy.py +++ b/tests/e2e/sre_authority/proxy.py @@ -141,18 +141,32 @@ def assert_filtered(secret): require(MARKER not in json.dumps(secret), "Secret material survived projection") +def secret_list_wire_facts(value, uid): + require(isinstance(value, dict) and value.get("kind") == "SecretList" + and isinstance(value.get("items"), list) and len(value["items"]) == 1 + and isinstance(value["items"][0], dict) and isinstance(value["items"][0].get("metadata"), dict) + and value["items"][0]["metadata"].get("uid") == uid, + "Native wire proof must select only the exact owned synthetic Secret") + item = value["items"][0] + return {"envelopeKind": "SecretList", "itemHasKind": "kind" in item, + "itemHasApiVersion": "apiVersion" in item, "itemMetadataObject": isinstance(item.get("metadata"), dict)} + + def proxy_acceptance(h): install_metrics(h) pod = alive_pinned_source(h) projection_and_files(h, pod) runtime_denials(h, pod["metadata"]["name"]) token_secret_denials(h) - h.create({"apiVersion": "v1", "kind": "Secret", + fixture = h.create({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": f"sre-filter-{h.phase}", "namespace": OPERATORS, "labels": {"copy": MARKER}, "annotations": {"kubectl.kubernetes.io/last-applied-configuration": json.dumps({"data": {"copy": MARKER}})}}, "stringData": {"operator-token": MARKER, "password": MARKER}}) name = f"sre-filter-{h.phase}" + native = h.api("GET", f"/api/v1/namespaces/{OPERATORS}/secrets?fieldSelector=metadata.name%3D{name}", + status=200).json() + print("SRE-WIRE", json.dumps(secret_list_wire_facts(native, fixture["metadata"]["uid"])), flush=True) with h.port_forward(pod["metadata"]["name"]) as port: kube_module, sre = load_unchanged_hermes(h, port) kube = kube_module.client() From 41820a2eb3f6116602706626886b4382c9ad77aa Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 22:40:29 +0200 Subject: [PATCH 45/62] test(e2e): diagnose Kubernetes log media without exposing logs Real Hermes GET/LIST filtering now passes. Record only the unchanged log reader's status/known rejection source and marker booleans, never log contents or error bodies. Compare text/plain, application/json and wildcard Accept against a bounded one-line log request on the owned Kind control-plane Pod. Production code and all normal CI dependencies remain unchanged; 84 Python tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../e2e/sre_authority/bootstrap_diagnostics.py | 6 ++++++ tests/e2e/sre_authority/bootstrap_probe.py | 9 +++++++++ tests/e2e/sre_authority/harness_test.py | 15 ++++++++++++++- tests/e2e/sre_authority/proxy.py | 18 ++++++++++++++++++ tests/e2e/sre_authority/registration_schema.py | 4 ++-- 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 17ab949a3..820a7c221 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -173,6 +173,12 @@ def exception_summary(error, root): body = response.json() except (ValueError, TypeError): return result + result.update(response_summary(status, body, root)) + return result + + +def response_summary(status, body, root): + result = {"httpStatus": status} if not isinstance(body, dict) or body.get("kind") != "Status": return result reason = body.get("reason") diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 79abbcf82..dac12db32 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -222,6 +222,15 @@ def main(root, diagnostics_only, candidate=False, retirement=False): CONTEXT, "kube-controller-manager-kars-e2e-control-plane")) else: try: + media = [] + path = ("/api/v1/namespaces/kube-system/pods/kube-controller-manager-kars-e2e-control-plane/log" + "?container=kube-controller-manager&tailLines=1&limitBytes=4096") + for accept in ("text/plain", "application/json", "*/*"): + code, _ = request(port, "GET", path, accept=accept) + media.append({"accept": accept, "httpStatus": code}) + write_report(root, "bootstrap-log-media.json", {"cases": media}) + if next(case["httpStatus"] for case in media if case["accept"] == "application/json") != 200: + raise RuntimeError("Actual API log media precondition failed") state = exercise(root, port, objects, policies, wait_seconds=180 if candidate else 90, retirement=retirement and not candidate) from sre_authority.bootstrap_cases import admission_cases diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index bda59356a..8d79ecb4d 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -20,7 +20,7 @@ ) from sre_authority.fixtures import seed_control_consumer from sre_authority.admission import reserved_source_probe -from sre_authority.proxy import MARKER, assert_filtered, secret_list_wire_facts +from sre_authority.proxy import MARKER, assert_filtered, log_reader_facts, secret_list_wire_facts from sre_authority.credential_paths import assert_token_type_immutable, assert_watch_result @@ -34,6 +34,19 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_log_reader_diagnostics_never_echo_log_text_or_error_body(self): + root = Path(__file__).resolve().parents[3] + facts = log_reader_facts(root, {"error": "406 Not Acceptable", + "body": json.dumps({"kind": "Status", "reason": "SREProxyDenied", + "message": "Kubernetes rejected the diagnostic/proposal request", "data": MARKER})}) + self.assertEqual(facts["httpStatus"], 406) + self.assertTrue(facts["hasError"]) + self.assertIn("responseSite", facts) + self.assertNotIn(MARKER, json.dumps(facts)) + facts = log_reader_facts(root, {"logs": "sre-standin-alive " + MARKER}) + self.assertTrue(facts["standinMarker"]) + self.assertNotIn(MARKER, json.dumps(facts)) + def test_native_secret_list_wire_facts_publish_no_synthetic_secret_fields(self): value = {"kind": "SecretList", "items": [{"metadata": {"uid": "fixture", "annotations": {"copy": MARKER}}, "data": {"token": MARKER}}]} diff --git a/tests/e2e/sre_authority/proxy.py b/tests/e2e/sre_authority/proxy.py index 0730ea8ae..0a9fdb6d7 100644 --- a/tests/e2e/sre_authority/proxy.py +++ b/tests/e2e/sre_authority/proxy.py @@ -4,6 +4,7 @@ import importlib.util import json import os +import re import sys import types @@ -152,6 +153,22 @@ def secret_list_wire_facts(value, uid): "itemHasApiVersion": "apiVersion" in item, "itemMetadataObject": isinstance(item.get("metadata"), dict)} +def log_reader_facts(root, value): + from .bootstrap_diagnostics import response_summary + text = value.get("logs") + facts = {"hasError": "error" in value, "hasText": isinstance(text, str), + "standinMarker": isinstance(text, str) and "sre-standin-alive" in text} + error = value.get("error") + status = re.match(r"^([1-5][0-9]{2}) ", error) if isinstance(error, str) else None + if status: + try: + body = json.loads(value.get("body", "")) + except (ValueError, TypeError): + body = None + facts.update(response_summary(int(status[1]), body, root)) + return facts + + def proxy_acceptance(h): install_metrics(h) pod = alive_pinned_source(h) @@ -178,6 +195,7 @@ def proxy_acceptance(h): assert_filtered(listing["items"][0]) h.passed("Unchanged Hermes HTTPS client GET/LIST retains Secret key names but no values or metadata copies") logs = sre._impl_sre_logs(namespace=RUNTIME, pod=pod["metadata"]["name"], container="agent", tail=20) + print("SRE-LOG-WIRE", json.dumps(log_reader_facts(h.root, logs)), flush=True) require("error" not in logs and "sre-standin-alive" in logs.get("logs", ""), "Unchanged Hermes raw log reader failed") metrics = kube.get("/apis/metrics.k8s.io/v1beta1/nodes") require(metrics.get("kind") == "NodeMetricsList" and metrics.get("items"), "Real metrics did not pass the filtered proxy") diff --git a/tests/e2e/sre_authority/registration_schema.py b/tests/e2e/sre_authority/registration_schema.py index 2b1f3326b..5b5d5ec47 100644 --- a/tests/e2e/sre_authority/registration_schema.py +++ b/tests/e2e/sre_authority/registration_schema.py @@ -111,10 +111,10 @@ def command(stage, args, *, root, data=None): return result.stdout -def request(port, method, path, obj=None): +def request(port, method, path, obj=None, *, accept="application/json"): body = None if obj is None else json.dumps(obj).encode() req = Request(f"http://127.0.0.1:{port}{path}", data=body, method=method, - headers={"Content-Type": "application/json", "Accept": "application/json"}) + headers={"Content-Type": "application/json", "Accept": accept}) opener = build_opener(ProxyHandler({})) try: response = opener.open(req, timeout=15) From 4b4a92a910ae23d7c16a5894ab457dd7703f6ba3 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 22:49:05 +0200 Subject: [PATCH 46/62] fix(sre): negotiate Kubernetes log streams with supported media Real Kind API proof: Accept text/plain returns 406 for Pod logs, while application/json and wildcard return 200. Use wildcard only on the bounded upstream log path; keep the facade's plain-text response, byte/query caps, private authority checks and all JSON/media boundaries unchanged. Add HTTPS regression reproducing the native 406 before verifying raw log delivery. Python: 84 passed; no local Cargo, normal CI dependencies retained. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- docs/how-to/sre-authority.md | 2 ++ inference-router/src/sre_proxy/backend.rs | 9 +----- inference-router/src/sre_proxy/tests.rs | 34 ++++++++++++++++++++++- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index 1f296179f..83541df77 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -204,6 +204,8 @@ Standard `KUBERNETES_SERVICE_HOST/PORT` point to `https://127.0.0.1:9446`. Pinned Hermes clients continue using HTTPS, CA verification, raw pod-log GETs, and proposal POSTs without an image-specific fallback. Azure token projection is excluded from the agent container. The old apiserver egress bypass is gone. +Upstream Pod-log requests use API-compatible media negotiation; the facade +still returns only the bounded plain-text log response. Admission protects both direct Pod mounts and Deployment/ReplicaSet/Job and CronJob templates from laundering a private mount through Kubernetes workload controllers. Exec/attach/port-forward into the private SRE runtime requires diff --git a/inference-router/src/sre_proxy/backend.rs b/inference-router/src/sre_proxy/backend.rs index c59ba510a..8d86ceaf5 100644 --- a/inference-router/src/sre_proxy/backend.rs +++ b/inference-router/src/sre_proxy/backend.rs @@ -374,14 +374,7 @@ impl Backend { format!("{}{}", self.config.kube_url.trim_end_matches('/'), path), ) .bearer_auth(self.bearer().await?) - .header( - "accept", - if logs { - "text/plain" - } else { - "application/json" - }, - ); + .header("accept", if logs { "*/*" } else { "application/json" }); if let Some(body) = body { request = request.json(&body); } diff --git a/inference-router/src/sre_proxy/tests.rs b/inference-router/src/sre_proxy/tests.rs index 1ec0ebf47..76b9ff90c 100644 --- a/inference-router/src/sre_proxy/tests.rs +++ b/inference-router/src/sre_proxy/tests.rs @@ -99,7 +99,12 @@ async fn fixture() -> Fixture { item.as_object_mut().unwrap().remove("apiVersion"); json!({"apiVersion":"v1","kind":"SecretList","metadata":{},"items":[item]}) } - "/api/v1/namespaces/kars-demo/pods/app/log" => return ResponseTemplate::new(200).set_body_raw("legitimate pod log\n","text/plain"), + "/api/v1/namespaces/kars-demo/pods/app/log" => { + if request.headers.get("accept").and_then(|value|value.to_str().ok()) != Some("*/*") { + return ResponseTemplate::new(406).set_body_json(json!({"kind":"Status","reason":"NotAcceptable"})); + } + return ResponseTemplate::new(200).set_body_raw("legitimate pod log\n","text/plain"); + } "/apis/metrics.k8s.io/v1beta1/nodes" => json!({"kind":"NodeMetricsList","items":[]}), "/apis/kars.azure.com/v1alpha1/namespaces/kars-sre/karssreactions" if request.method=="POST" => { let body:serde_json::Value=request.body_json().unwrap(); @@ -230,6 +235,33 @@ fn secret() -> serde_json::Value { "data":{"control-token":PRIVATE_VALUE},"stringData":{"copy":PRIVATE_VALUE}}) } +#[tokio::test] +async fn upstream_log_negotiation_keeps_api_compatible_accept_and_bounded_plain_text() { + let f = fixture().await; + let path = "/api/v1/namespaces/kars-demo/pods/app/log"; + let rejected = reqwest::Client::new() + .get(format!("{}{path}", f.backend.config.kube_url)) + .bearer_auth("private-kubernetes-token") + .header("accept", "text/plain") + .send() + .await + .unwrap(); + assert_eq!(rejected.status(), StatusCode::NOT_ACCEPTABLE); + let response = f + .client + .get(format!("{}{path}?tailLines=20", f.url)) + .bearer_auth(&f.token) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()["content-type"], + "text/plain; charset=utf-8" + ); + assert_eq!(response.text().await.unwrap(), "legitimate pod log\n"); +} + #[tokio::test] async fn agent_credential_cannot_read_control_material_directly_or_through_tls_proxy() { let f = fixture().await; From e2b31f5f8cf589044f21de584f961ee198cdfcf6 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Wed, 9 Sep 2026 23:50:12 +0200 Subject: [PATCH 47/62] test(e2e): prove guarded canonical consumer cleanup authority Real legacy acceptance now passes reads, logs, metrics, Pending proposals and denial paths, reaches Retired, then stalls deleting the namespace with Deployment/sre still present. Add sanitized namespace conditions and real UID/RV-fenced DELETE dry-runs comparing the existing namespace-controller and Kars controller principals. Preserve the consumer guard and grant no new permissions. Python: 85 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/bootstrap_cases.py | 60 ++++++++++++++++++- tests/e2e/sre_authority/bootstrap_probe.py | 6 +- .../e2e/sre_authority/bootstrap_probe_test.py | 21 +++++++ tests/e2e/sre_authority/common.py | 3 +- 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index 9db6cc47f..73fb62764 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -17,8 +17,8 @@ DEPLOYMENT_CONTROLLER = "system:serviceaccount:kube-system:deployment-controller" -def as_tenant(port, path, obj, *, user=USER): - req = Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(obj).encode(), method="POST", +def as_tenant(port, path, obj, *, user=USER, method="POST"): + req = Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(obj).encode(), method=method, headers={"Content-Type": "application/json", "Accept": "application/json", "Impersonate-User": user}) try: @@ -207,3 +207,59 @@ def private_controller_chain(port, policies, report): time.sleep(0.5) report(snapshot) raise RuntimeError("Actual private Deployment/ReplicaSet controllers did not create the admission-only Pod") + + +def namespace_cleanup_cases(port, policies): + path = "/apis/apps/v1/namespaces/kars-sre/deployments/sre" + code, _ = request(port, "GET", path) + if code != 404: + raise RuntimeError("Namespace cleanup proof refuses an existing canonical Deployment") + obj = {"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": "sre", "namespace": "kars-sre"}, + "spec": {"replicas": 0, "selector": {"matchLabels": {"app": "e2e-retired-consumer"}}, + "template": {"metadata": {"labels": {"app": "e2e-retired-consumer"}}, "spec": { + "automountServiceAccountToken": False, "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]}}}} + code, created = request(port, "POST", path.rsplit("/", 1)[0], obj) + if code != 201 or not created.get("metadata", {}).get("uid"): + raise RuntimeError("Canonical no-execution cleanup fixture CREATE failed") + uid = created["metadata"]["uid"] + reports = [] + primary_failure = False + try: + for account, namespace, expected in (("namespace-controller", "kube-system", 403), + ("kars-controller", "kars-system", 200)): + principal = f"system:serviceaccount:{namespace}:{account}" + code, sa = request(port, "GET", f"/api/v1/namespaces/{namespace}/serviceaccounts/{account}") + if code != 200 or not sa.get("metadata", {}).get("uid"): + raise RuntimeError("Cleanup proof principal does not exist") + code, current = request(port, "GET", path) + if code != 200 or current.get("metadata", {}).get("uid") != uid: + raise RuntimeError("Cleanup proof Deployment was replaced before its dry-run") + code, response = as_tenant(port, path + "?dryRun=All", { + "apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, + user=principal, method="DELETE") + result = api_result(code, response, policies) + result.update({"case": f"{account}-canonical-consumer-delete", "expectedStatus": expected, + "matched": code == expected and (code != 403 + or "kars-sre-consumer-authority" in result.get("policies", []))}) + reports.append(result) + return reports + except BaseException: + primary_failure = True + raise + finally: + try: + code, current = request(port, "GET", path) + if code != 200 or current.get("metadata", {}).get("uid") != uid: + raise RuntimeError("Cleanup proof Deployment identity changed") + code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}) + if code not in (200, 202): + raise RuntimeError("Cleanup proof could not remove its owned Deployment") + except Exception: + if not primary_failure: + raise + print("SRE-DIAG owned canonical cleanup fixture removal unavailable", flush=True) diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index dac12db32..f62da1b12 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -242,13 +242,17 @@ def main(root, diagnostics_only, candidate=False, retirement=False): from sre_authority.binding_probe import prove prove(root, port, state, objects, lambda facts: write_report(root, "bootstrap-binding-retirement.json", facts)) - from sre_authority.bootstrap_cases import deployment_controller_cases, private_controller_chain + from sre_authority.bootstrap_cases import deployment_controller_cases, namespace_cleanup_cases, private_controller_chain controller_cases = deployment_controller_cases(port, policies) write_report(root, "bootstrap-workload-controller.json", {"cases": controller_cases}) if not all(case["matched"] for case in controller_cases): raise RuntimeError("Built-in Deployment controller cannot create the private SRE ReplicaSet") private_controller_chain(port, policies, lambda facts: write_report(root, "bootstrap-private-controller-chain.json", facts)) + cleanup_cases = namespace_cleanup_cases(port, policies) + write_report(root, "bootstrap-namespace-cleanup.json", {"cases": cleanup_cases}) + if not all(case["matched"] for case in cleanup_cases): + raise RuntimeError("Canonical consumer cleanup authority differs from the expected boundary") finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index e0d2b01b7..34e18cf6d 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -15,6 +15,27 @@ class BootstrapProofTests(unittest.TestCase): + def test_namespace_cleanup_proof_retains_guard_and_fences_all_deletes(self): + from sre_authority.bootstrap_cases import namespace_cleanup_cases + deployment = {"metadata": {"uid": "owned", "resourceVersion": "2"}} + responses = [(404, {}), (201, deployment), (200, {"metadata": {"uid": "namespace-controller"}}), + (200, deployment), (200, {"metadata": {"uid": "kars-controller"}}), + (200, deployment), (200, deployment), (200, {})] + with patch("sre_authority.bootstrap_cases.request", side_effect=responses) as api, \ + patch("sre_authority.bootstrap_cases.as_tenant", side_effect=[ + (403, {"kind": "Status", "reason": "Forbidden", + "message": "kars-sre-consumer-authority denied do-not-publish"}), + (200, {"kind": "Status", "status": "Success"})]) as actor: + cases = namespace_cleanup_cases(1, {"kars-sre-consumer-authority": {}}) + self.assertTrue(all(case["matched"] for case in cases)) + self.assertNotIn("do-not-publish", json.dumps(cases)) + for call in actor.call_args_list: + self.assertEqual(call.kwargs["method"], "DELETE") + self.assertTrue(call.args[1].endswith("?dryRun=All")) + self.assertEqual(call.args[2]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) + self.assertEqual(api.call_args_list[-1].args[1], "DELETE") + self.assertEqual(api.call_args_list[-1].args[3]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) + def test_http_failure_summary_reports_status_and_checked_source_not_body_url_or_headers(self): root = Path(__file__).resolve().parents[3] class Response: diff --git a/tests/e2e/sre_authority/common.py b/tests/e2e/sre_authority/common.py index eec1cfae7..582a3332c 100644 --- a/tests/e2e/sre_authority/common.py +++ b/tests/e2e/sre_authority/common.py @@ -378,7 +378,8 @@ def diagnostics(self): self.deadline = max(self.deadline, time.monotonic() + 90) # Status and identities only; never dump Secret bodies or whole Pods. for kind, name, namespace in [("karssreregistrations.kars.azure.com", "canonical", None), - ("karssandbox", "sre", SYSTEM), ("deployment", "sre", RUNTIME)]: + ("karssandbox", "sre", SYSTEM), ("deployment", "sre", RUNTIME), + ("namespace", RUNTIME, None)]: try: obj = self.get(kind, name, namespace) if obj: From 38aba9618d4b7a3f990e97ab6c5ef518af0ad0a2 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 00:01:03 +0200 Subject: [PATCH 48/62] test(e2e): hand off only the owned consumer to cleanup proof The prior binding experiment already owns the canonical no-execution consumer. Reuse it only with exact captured UID and unchanged spec after that experiment completes; never adopt an arbitrary existing Deployment. Extend safe failure coordinates to the case module. Production cleanup candidate remains local pending actual API proof. Python: 86 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/bootstrap_cases.py | 18 ++++++++++++------ tests/e2e/sre_authority/bootstrap_probe.py | 4 ++-- .../e2e/sre_authority/bootstrap_probe_test.py | 11 +++++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index 73fb62764..073249c9f 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -209,10 +209,15 @@ def private_controller_chain(port, policies, report): raise RuntimeError("Actual private Deployment/ReplicaSet controllers did not create the admission-only Pod") -def namespace_cleanup_cases(port, policies): +def namespace_cleanup_cases(port, policies, owned_consumer=None): path = "/apis/apps/v1/namespaces/kars-sre/deployments/sre" - code, _ = request(port, "GET", path) - if code != 404: + code, current = request(port, "GET", path) + if owned_consumer is not None: + if (code != 200 or current.get("metadata", {}).get("uid") != owned_consumer["metadata"]["uid"] + or current.get("spec") != owned_consumer.get("spec")): + raise RuntimeError("Earlier owned cleanup fixture changed; no adoption permitted") + created = current + elif code != 404: raise RuntimeError("Namespace cleanup proof refuses an existing canonical Deployment") obj = {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "sre", "namespace": "kars-sre"}, @@ -221,9 +226,10 @@ def namespace_cleanup_cases(port, policies): "automountServiceAccountToken": False, "schedulerName": "kars-e2e-admission-never-schedule", "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", "imagePullPolicy": "Never"}]}}}} - code, created = request(port, "POST", path.rsplit("/", 1)[0], obj) - if code != 201 or not created.get("metadata", {}).get("uid"): - raise RuntimeError("Canonical no-execution cleanup fixture CREATE failed") + if owned_consumer is None: + code, created = request(port, "POST", path.rsplit("/", 1)[0], obj) + if code != 201 or not created.get("metadata", {}).get("uid"): + raise RuntimeError("Canonical no-execution cleanup fixture CREATE failed") uid = created["metadata"]["uid"] reports = [] primary_failure = False diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index f62da1b12..96f1f18ee 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -26,7 +26,7 @@ def failure_site(error): frame = error.__traceback__ while frame: name = Path(frame.tb_frame.f_code.co_filename).name - if name in ("bootstrap_probe.py", "binding_probe.py"): + if name in ("bootstrap_probe.py", "binding_probe.py", "bootstrap_cases.py"): result.update(source=name, line=frame.tb_lineno) frame = frame.tb_next return result @@ -249,7 +249,7 @@ def main(root, diagnostics_only, candidate=False, retirement=False): raise RuntimeError("Built-in Deployment controller cannot create the private SRE ReplicaSet") private_controller_chain(port, policies, lambda facts: write_report(root, "bootstrap-private-controller-chain.json", facts)) - cleanup_cases = namespace_cleanup_cases(port, policies) + cleanup_cases = namespace_cleanup_cases(port, policies, state["consumer"] if state else None) write_report(root, "bootstrap-namespace-cleanup.json", {"cases": cleanup_cases}) if not all(case["matched"] for case in cleanup_cases): raise RuntimeError("Canonical consumer cleanup authority differs from the expected boundary") diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 34e18cf6d..95e107b01 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -36,6 +36,17 @@ def test_namespace_cleanup_proof_retains_guard_and_fences_all_deletes(self): self.assertEqual(api.call_args_list[-1].args[1], "DELETE") self.assertEqual(api.call_args_list[-1].args[3]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) + def test_cleanup_proof_never_adopts_an_unexpected_existing_consumer(self): + from sre_authority.bootstrap_cases import namespace_cleanup_cases + owned = {"metadata": {"uid": "owned"}, "spec": {"replicas": 0}} + for current, expected in ((owned, None), + ({"metadata": {"uid": "other"}, "spec": owned["spec"]}, owned), + ({"metadata": owned["metadata"], "spec": {"replicas": 1}}, owned)): + with patch("sre_authority.bootstrap_cases.request", return_value=(200, current)) as api, \ + self.assertRaises(RuntimeError): + namespace_cleanup_cases(1, {}, expected) + api.assert_called_once() + def test_http_failure_summary_reports_status_and_checked_source_not_body_url_or_headers(self): root = Path(__file__).resolve().parents[3] class Response: From f979d1fb55732b6f92837eb3c58ad46f3a7918a7 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 00:23:03 +0200 Subject: [PATCH 49/62] test(e2e): qualify ReplicaSet updates and forged owner references Add actual dry-run CREATE/UPDATE cases for namespaced tenant attempts using the approved private parent's real UID, private ReplicaSet owner-reference edits, trusted Deployment-controller updates and private parent Deployment updates. Retain all existing tenant negative and real built-in UID-chain cases. Fix DELETE dry-run proof to set DeleteOptions.dryRun in the body. No production code or policy changes in this commit; pending cleanup repair remains local for parent review. Python: 89 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/bootstrap_cases.py | 8 +- tests/e2e/sre_authority/bootstrap_probe.py | 9 +- .../e2e/sre_authority/bootstrap_probe_test.py | 3 +- .../sre_authority/controller_update_probe.py | 98 +++++++++++++++++++ .../controller_update_probe_test.py | 85 ++++++++++++++++ 5 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/sre_authority/controller_update_probe.py create mode 100644 tests/e2e/sre_authority/controller_update_probe_test.py diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index 073249c9f..84c9b61b7 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -42,7 +42,7 @@ def admission_cases(port, policies): {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, "rules": [{"apiGroups": [""], "resources": ["pods"], "verbs": ["create"]}, - {"apiGroups": ["apps"], "resources": ["deployments", "replicasets"], "verbs": ["create"]}, + {"apiGroups": ["apps"], "resources": ["deployments", "replicasets"], "verbs": ["create", "update"]}, {"apiGroups": ["kars.azure.com"], "resources": ["karssreactions"], "verbs": ["create"]}]}, {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, @@ -203,7 +203,7 @@ def private_controller_chain(port, policies, report): report(snapshot) if not snapshot["privateMountPreserved"] or not snapshot["noWorkloadExecution"]: raise RuntimeError("Private controller-chain proof changed its protected template or executed a workload") - return + return created time.sleep(0.5) report(snapshot) raise RuntimeError("Actual private Deployment/ReplicaSet controllers did not create the admission-only Pod") @@ -243,8 +243,8 @@ def namespace_cleanup_cases(port, policies, owned_consumer=None): code, current = request(port, "GET", path) if code != 200 or current.get("metadata", {}).get("uid") != uid: raise RuntimeError("Cleanup proof Deployment was replaced before its dry-run") - code, response = as_tenant(port, path + "?dryRun=All", { - "apiVersion": "v1", "kind": "DeleteOptions", + code, response = as_tenant(port, path, { + "apiVersion": "v1", "kind": "DeleteOptions", "dryRun": ["All"], "preconditions": {"uid": uid, "resourceVersion": current["metadata"]["resourceVersion"]}}, user=principal, method="DELETE") result = api_result(code, response, policies) diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 96f1f18ee..0ee990762 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -26,7 +26,7 @@ def failure_site(error): frame = error.__traceback__ while frame: name = Path(frame.tb_frame.f_code.co_filename).name - if name in ("bootstrap_probe.py", "binding_probe.py", "bootstrap_cases.py"): + if name in ("bootstrap_probe.py", "binding_probe.py", "bootstrap_cases.py", "controller_update_probe.py"): result.update(source=name, line=frame.tb_lineno) frame = frame.tb_next return result @@ -247,8 +247,13 @@ def main(root, diagnostics_only, candidate=False, retirement=False): write_report(root, "bootstrap-workload-controller.json", {"cases": controller_cases}) if not all(case["matched"] for case in controller_cases): raise RuntimeError("Built-in Deployment controller cannot create the private SRE ReplicaSet") - private_controller_chain(port, policies, + parent = private_controller_chain(port, policies, lambda facts: write_report(root, "bootstrap-private-controller-chain.json", facts)) + from sre_authority.controller_update_probe import cases as update_cases + updates = update_cases(port, policies, parent) + write_report(root, "bootstrap-controller-update-ownerrefs.json", {"cases": updates}) + if not all(case["matched"] for case in updates): + raise RuntimeError("Private workload UPDATE/owner-reference boundary failed") cleanup_cases = namespace_cleanup_cases(port, policies, state["consumer"] if state else None) write_report(root, "bootstrap-namespace-cleanup.json", {"cases": cleanup_cases}) if not all(case["matched"] for case in cleanup_cases): diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index 95e107b01..a9d87de1e 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -31,7 +31,8 @@ def test_namespace_cleanup_proof_retains_guard_and_fences_all_deletes(self): self.assertNotIn("do-not-publish", json.dumps(cases)) for call in actor.call_args_list: self.assertEqual(call.kwargs["method"], "DELETE") - self.assertTrue(call.args[1].endswith("?dryRun=All")) + self.assertNotIn("?", call.args[1]) + self.assertEqual(call.args[2]["dryRun"], ["All"]) self.assertEqual(call.args[2]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) self.assertEqual(api.call_args_list[-1].args[1], "DELETE") self.assertEqual(api.call_args_list[-1].args[3]["preconditions"], {"uid": "owned", "resourceVersion": "2"}) diff --git a/tests/e2e/sre_authority/controller_update_probe.py b/tests/e2e/sre_authority/controller_update_probe.py new file mode 100644 index 000000000..536cf610f --- /dev/null +++ b/tests/e2e/sre_authority/controller_update_probe.py @@ -0,0 +1,98 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Actual UPDATE/owner-reference checks for the narrow ReplicaSet capability.""" + +import copy + +from .bootstrap_cases import DEPLOYMENT_CONTROLLER, USER, as_tenant +from .bootstrap_diagnostics import api_result +from .common import require +from .registration_schema import request + + +def cases(port, policies, parent): + namespace = "kars-sre" + parent_path = f"/apis/apps/v1/namespaces/{namespace}/deployments/{parent['metadata']['name']}" + code, current = request(port, "GET", parent_path) + require(code == 200 and current["metadata"]["uid"] == parent["metadata"]["uid"] + and current["spec"] == parent["spec"], "Approved private parent changed before update proof") + owner = {"apiVersion": "apps/v1", "kind": "Deployment", "name": parent["metadata"]["name"], + "uid": parent["metadata"]["uid"], "controller": True} + private_pod = copy.deepcopy(parent["spec"]["template"]["spec"]) + ordinary_pod = copy.deepcopy(private_pod) + ordinary_pod.pop("volumes", None) + for container in ordinary_pod["containers"]: + container.pop("volumeMounts", None) + paths = "/apis/apps/v1/namespaces/kars-sre/replicasets" + fixtures, reports = [], [] + primary_failure = False + try: + for suffix, template in (("ordinary", ordinary_pod), ("private", private_pod)): + name = f"e2e-update-{suffix}" + selector = {"app": name} + obj = {"apiVersion": "apps/v1", "kind": "ReplicaSet", + "metadata": {"name": name, "namespace": namespace}, + "spec": {"replicas": 0, "selector": {"matchLabels": selector}, + "template": {"metadata": {"labels": selector}, "spec": template}}} + code, created = request(port, "POST", paths, obj) + require(code == 201 and created.get("metadata", {}).get("uid"), + "Owned update fixture CREATE failed") + fixtures.append((f"{paths}/{name}", created)) + ordinary_path, ordinary = fixtures[0] + private_path, private = fixtures[1] + probes = [ + ("tenant-private-rs-forged-parent-create", USER, "POST", paths, private, True, True, 403), + ("tenant-ordinary-rs-update", USER, "PUT", ordinary_path, ordinary, False, False, 200), + ("tenant-private-rs-forged-parent-update", USER, "PUT", ordinary_path, ordinary, True, True, 403), + ("tenant-owned-private-rs-ownerref-update", USER, "PUT", private_path, private, True, True, 403), + ("deployment-controller-private-rs-update", DEPLOYMENT_CONTROLLER, "PUT", private_path, private, True, True, 200), + ("tenant-private-parent-update", USER, "PUT", parent_path, parent, True, False, 403), + ] + for label, principal, method, path, original, is_private, reference, expected in probes: + if method == "PUT": + code, obj = request(port, "GET", path) + require(code == 200 and obj["metadata"]["uid"] == original["metadata"]["uid"], + "Update proof target UID changed") + obj = copy.deepcopy(obj) + else: + obj = copy.deepcopy(original) + obj["metadata"] = {"name": "e2e-forged-private-create", "namespace": namespace} + obj.pop("status", None) + if is_private: + obj["spec"]["template"]["spec"] = copy.deepcopy(private_pod) + if reference: + obj["metadata"]["ownerReferences"] = [owner] + obj["spec"]["template"].setdefault("metadata", {}).setdefault("annotations", {})["e2e-update-proof"] = "true" + code, body = as_tenant(port, path + "?dryRun=All", obj, user=principal, method=method) + result = api_result(code, body, policies) + result.update({"case": label, "expectedStatus": expected, + "matched": code == expected and (expected != 403 + or "kars-sre-private-workloads" in result.get("policies", [])) + and (expected == 403 or body.get("kind") == obj["kind"] + and body.get("metadata", {}).get("uid") == original["metadata"]["uid"])}) + reports.append(result) + for path, original in fixtures + [(parent_path, parent)]: + code, current = request(port, "GET", path) + require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"] + and current["spec"] == original["spec"] + and current["metadata"].get("ownerReferences") == original["metadata"].get("ownerReferences"), + "Owner/update dry-run changed a real workload") + return reports + except BaseException: + primary_failure = True + raise + finally: + for path, original in reversed(fixtures): + try: + code, current = request(port, "GET", path) + require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"], + "Update fixture cleanup identity changed") + code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}}) + require(code in (200, 202), "Owned update fixture cleanup failed") + except Exception: + if not primary_failure: + raise + print("SRE-DIAG Owned update fixture cleanup unavailable", flush=True) diff --git a/tests/e2e/sre_authority/controller_update_probe_test.py b/tests/e2e/sre_authority/controller_update_probe_test.py new file mode 100644 index 000000000..27e0d0346 --- /dev/null +++ b/tests/e2e/sre_authority/controller_update_probe_test.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +import unittest +from unittest.mock import patch + +from sre_authority.bootstrap_cases import DEPLOYMENT_CONTROLLER, USER +from sre_authority.controller_update_probe import cases + + +class ControllerUpdateProofTests(unittest.TestCase): + def fixture(self): + parent = {"apiVersion": "apps/v1", "kind": "Deployment", + "metadata": {"name": "approved-parent", "namespace": "kars-sre", "uid": "parent", "resourceVersion": "1"}, + "spec": {"replicas": 0, "template": {"spec": { + "containers": [{"name": "probe", "image": "registry.invalid/probe:never", + "volumeMounts": [{"name": "private", "mountPath": "/private"}]}], + "volumes": [{"name": "private", "secret": {"secretName": "sre-api-router-identity"}}]}}}} + path = "/apis/apps/v1/namespaces/kars-sre/deployments/approved-parent" + objects = {path: copy.deepcopy(parent)} + calls = [] + def api(_port, method, path, obj=None): + calls.append((method, path, copy.deepcopy(obj))) + if method == "GET": + return 200, copy.deepcopy(objects[path]) + if method == "POST": + created = copy.deepcopy(obj) + created["metadata"].update(uid=created["metadata"]["name"], resourceVersion="1") + objects[path + "/" + created["metadata"]["name"]] = created + return 201, copy.deepcopy(created) + if method == "DELETE": + self.assertEqual(obj["preconditions"], { + "uid": objects[path]["metadata"]["uid"], "resourceVersion": objects[path]["metadata"]["resourceVersion"]}) + del objects[path] + return 200, {"kind": "Status"} + raise AssertionError("Unexpected API mutation") + return parent, objects, calls, api + + def test_update_ownerrefs_use_actual_parent_and_never_persist_dry_runs(self): + parent, objects, operations, api = self.fixture() + outcomes = iter([403, 200, 403, 403, 200, 403]) + probes = [] + def actor(_port, path, obj, **kwargs): + probes.append((path, copy.deepcopy(obj), kwargs)) + code = next(outcomes) + return code, (copy.deepcopy(obj) if code == 200 else { + "kind": "Status", "reason": "Forbidden", + "message": "kars-sre-private-workloads denied do-not-publish"}) + with patch("sre_authority.controller_update_probe.request", side_effect=api), \ + patch("sre_authority.controller_update_probe.as_tenant", side_effect=actor): + result = cases(1, {"kars-sre-private-workloads": {}}, parent) + self.assertTrue(all(case["matched"] for case in result)) + self.assertNotIn("do-not-publish", json.dumps(result)) + self.assertTrue(all(path.endswith("?dryRun=All") for path, _, _ in probes)) + self.assertEqual(probes[0][2], {"user": USER, "method": "POST"}) + self.assertEqual(probes[4][2], {"user": DEPLOYMENT_CONTROLLER, "method": "PUT"}) + self.assertEqual(probes[5][1]["kind"], "Deployment") + for index in (0, 2, 3, 4): + self.assertEqual(probes[index][1]["metadata"]["ownerReferences"], [{ + "apiVersion": "apps/v1", "kind": "Deployment", "name": "approved-parent", + "uid": "parent", "controller": True}]) + self.assertIn("volumes", probes[index][1]["spec"]["template"]["spec"]) + self.assertNotIn("volumes", probes[1][1]["spec"]["template"]["spec"]) + self.assertEqual(len(objects), 1) + self.assertEqual(sum(method == "DELETE" for method, _, _ in operations), 2) + + def test_unrelated_denial_or_missing_endpoint_is_not_an_admission_pass(self): + for code in (403, 404, 422): + parent, _, _, api = self.fixture() + with patch("sre_authority.controller_update_probe.request", side_effect=api), \ + patch("sre_authority.controller_update_probe.as_tenant", + return_value=(code, {"kind": "Status", "reason": "Forbidden", "message": "unrelated"})): + result = cases(1, {"kars-sre-private-workloads": {}}, parent) + self.assertFalse(any(case["matched"] for case in result)) + + def test_replaced_or_changed_approved_parent_stops_before_fixture_creation(self): + parent, _, _, _ = self.fixture() + for changed in ({"metadata": {"uid": "other"}, "spec": parent["spec"]}, + {"metadata": parent["metadata"], "spec": {}}): + with patch("sre_authority.controller_update_probe.request", return_value=(200, changed)) as api, \ + self.assertRaises(AssertionError): + cases(1, {}, parent) + api.assert_called_once() From 68169568a340aaadd30d1265b6fd39a3f98e1231 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 00:58:16 +0200 Subject: [PATCH 50/62] fix(sre): remove only the quiesced owned consumer before retirement Real Kind reaches Retired but namespace cleanup stalls with Deployment/sre retained. Real API DELETE proof confirms namespace-controller is correctly denied while the existing registrar-authorized Kars controller is allowed. Delete only the captured UID-owned, still-owned, replicas-zero Deployment with current UID/RV preconditions after Pod quiescence; wait for actual removal and preserve foreign namespaces, replacements, ownership changes and finalizers. Already-Retired audit records can finish this owned cleanup without reissuing authority. Add focused retirement race/error/idempotence tests; no new RBAC or admission exception. Python 89 pass; Rust validation remains hosted with no local Cargo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- controller/src/sre_authority.rs | 1 + controller/src/sre_authority/migration.rs | 86 +++++++- .../src/sre_authority/retirement_tests.rs | 200 +++++++++++++++++- docs/how-to/sre-authority.md | 5 + 4 files changed, 286 insertions(+), 6 deletions(-) diff --git a/controller/src/sre_authority.rs b/controller/src/sre_authority.rs index fa42b5c80..43bc5e191 100644 --- a/controller/src/sre_authority.rs +++ b/controller/src/sre_authority.rs @@ -179,6 +179,7 @@ async fn reconcile_inner(client: &Client, reg: &KarsSRERegistration) -> Result<( && status.privacy_revision.as_deref() == Some(crate::sre_privacy::REVISION) }) { + migration::stop_registered_consumer_for_retirement(client, reg).await?; return check_secret_denial(client, ®.spec.runtime_namespace.name).await; } if !reg.spec.enabled { diff --git a/controller/src/sre_authority/migration.rs b/controller/src/sre_authority/migration.rs index a0f9cbe4d..79e2a4dc1 100644 --- a/controller/src/sre_authority/migration.rs +++ b/controller/src/sre_authority/migration.rs @@ -12,7 +12,7 @@ use k8s_openapi::api::{ }; use kube::{ Api, Client, ResourceExt, - api::{ListParams, Patch, PatchParams}, + api::{DeleteParams, ListParams, Patch, PatchParams, Preconditions}, }; use serde_json::json; @@ -23,11 +23,16 @@ const WAITING_FOR_ROTATION: &str = "Waiting for owned control credential consumers to restart on the new privacy epoch"; const WAITING_FOR_ROLLOUT: &str = "Owned control credential consumer has not completed its privacy-epoch rollout"; +const WAITING_FOR_CONSUMER_REMOVAL: &str = + "Waiting for the owned SRE consumer Deployment to be removed"; pub(super) fn is_waiting(detail: &str) -> bool { matches!( detail, - WAITING_FOR_CONSUMERS | WAITING_FOR_ROTATION | WAITING_FOR_ROLLOUT + WAITING_FOR_CONSUMERS + | WAITING_FOR_ROTATION + | WAITING_FOR_ROLLOUT + | WAITING_FOR_CONSUMER_REMOVAL ) } @@ -46,6 +51,12 @@ pub(super) async fn stop_registered_consumer_for_retirement( if namespace.metadata.uid.as_deref() != Some(reg.spec.runtime_namespace.uid.as_str()) { return Ok(()); } + let registration_uid = reg + .metadata + .uid + .as_deref() + .filter(|uid| !uid.is_empty()) + .ok_or("SRE retirement requires the actual registration UID")?; let api: Api = Api::namespaced(client.clone(), RUNTIME_NAMESPACE); let Some(deployment) = api .get_opt("sre") @@ -60,7 +71,8 @@ pub(super) async fn stop_registered_consumer_for_retirement( .and_then(|s| s.template.metadata.as_ref()) .and_then(|m| m.annotations.as_ref()) .and_then(|a| a.get(OWNER)) - == reg.metadata.uid.as_ref(); + .map(String::as_str) + == Some(registration_uid); if !ours && reg .status @@ -70,7 +82,73 @@ pub(super) async fn stop_registered_consumer_for_retirement( { return Ok(()); } - stop_legacy_consumer(client, reg).await + stop_legacy_consumer(client, reg).await?; + if !ours { + return Ok(()); + } + // Namespace-controller intentionally lacks registrar power. Remove only + // our stopped Deployment here instead of leaving protected content to GC. + let Some(current) = api + .get_opt("sre") + .await + .map_err(|error| api_error("Read stopped SRE consumer", error))? + else { + return Ok(()); + }; + let uid = deployment + .metadata + .uid + .as_deref() + .filter(|uid| !uid.is_empty()) + .ok_or("Retiring SRE consumer UID is missing")?; + let still_owned = current + .spec + .as_ref() + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|metadata| metadata.annotations.as_ref()) + .and_then(|annotations| annotations.get(OWNER)) + .map(String::as_str) + == Some(registration_uid); + if current.metadata.uid.as_deref() != Some(uid) + || !still_owned + || current.spec.as_ref().and_then(|spec| spec.replicas) != Some(0) + { + return Err("Retiring SRE consumer changed after quiescence; preserved".into()); + } + if current.metadata.deletion_timestamp.is_some() { + return Err(WAITING_FOR_CONSUMER_REMOVAL.into()); + } + let version = current + .metadata + .resource_version + .filter(|version| !version.is_empty()) + .ok_or("Retiring SRE consumer resourceVersion is missing")?; + match api + .delete( + "sre", + &DeleteParams { + preconditions: Some(Preconditions { + uid: Some(uid.into()), + resource_version: Some(version), + }), + ..Default::default() + }, + ) + .await + { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(api_error("Remove stopped owned SRE consumer", error)), + } + if api + .get_opt("sre") + .await + .map_err(|error| api_error("Verify owned SRE consumer removal", error))? + .is_some() + { + return Err(WAITING_FOR_CONSUMER_REMOVAL.into()); + } + Ok(()) } fn current_boundary(deployment: &Deployment, reg: &KarsSRERegistration) -> bool { diff --git a/controller/src/sre_authority/retirement_tests.rs b/controller/src/sre_authority/retirement_tests.rs index e1b4a2b51..cdc709f21 100644 --- a/controller/src/sre_authority/retirement_tests.rs +++ b/controller/src/sre_authority/retirement_tests.rs @@ -3,9 +3,10 @@ use super::privacy_tests::admission_ready; use super::tests::{State, fixture, registration}; -use super::{bindings, reconcile}; +use super::{bindings, migration, reconcile}; use crate::sre_registration::{ - BindingReview, ConsumerReview, KarsSRERegistration, RUNTIME_NAMESPACE, + BindingReview, ConsumerReview, EPOCH, KarsSRERegistration, OWNER, RUNTIME_NAMESPACE, + RegistrationStatus, }; use k8s_openapi::api::rbac::v1::{ClusterRoleBinding, RoleBinding}; use kube::{ @@ -20,6 +21,201 @@ const CRBS: &str = "/apis/rbac.authorization.k8s.io/v1/clusterrolebindings"; const CONSUMER: &str = "/apis/apps/v1/namespaces/kars-sre/deployments/sre"; const RETIRED: &str = "kars.azure.com/sre-legacy-retired"; +fn stopped_owned_consumer(reg: &KarsSRERegistration) -> Value { + json!({"apiVersion":"apps/v1","kind":"Deployment", + "metadata":{"name":"sre","namespace":RUNTIME_NAMESPACE,"uid":"owned-consumer","resourceVersion":"5"}, + "spec":{"replicas":0,"selector":{"matchLabels":{"app":"sre"}}, + "template":{"metadata":{"annotations":{OWNER:reg.metadata.uid,EPOCH:reg.epoch()}}, + "spec":{"containers":[],"serviceAccountName":"sandbox"}}}}) +} + +fn seed_stopped_consumer(state: &Arc>, reg: &KarsSRERegistration) { + let mut locked = state.lock().unwrap(); + locked + .objects + .insert(CONSUMER.into(), stopped_owned_consumer(reg)); + locked.objects.insert( + "/api/v1/namespaces/kars-sre/pods".into(), + json!({"apiVersion":"v1","kind":"PodList","metadata":{},"items":[]}), + ); +} + +#[tokio::test] +async fn disabled_retirement_removes_only_quiesced_owned_deployment_with_uid_rv_fences() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap(); + migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap(); + let locked = state.lock().unwrap(); + assert!(!locked.objects.contains_key(CONSUMER)); + let deletes: Vec<_> = locked + .calls + .iter() + .filter(|(method, _, _)| method == "DELETE") + .collect(); + assert_eq!(deletes.len(), 1); + assert_eq!(deletes[0].1, CONSUMER); + assert_eq!( + deletes[0].2["preconditions"], + json!({"uid":"owned-consumer","resourceVersion":"5"}) + ); +} + +#[tokio::test] +async fn retired_audit_record_repairs_its_leftover_owned_consumer_without_reissuing_authority() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + reg.status = Some(RegistrationStatus { + phase: "Retired".into(), + observed_generation: reg.metadata.generation.unwrap_or_default(), + privacy_revision: Some(crate::sre_privacy::REVISION.into()), + ..Default::default() + }); + seed_stopped_consumer(&state, ®); + reconcile(&client, ®).await.unwrap(); + let locked = state.lock().unwrap(); + assert!(!locked.objects.contains_key(CONSUMER)); + assert!(locked.calls.iter().all(|(method, path, _)| method == "GET" + || method == "DELETE" + || method == "POST" && path.ends_with("/subjectaccessreviews"))); +} + +#[tokio::test] +async fn retirement_preserves_foreign_namespace_or_unowned_consumer() { + for foreign_namespace in [true, false] { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + { + let mut locked = state.lock().unwrap(); + if foreign_namespace { + locked.namespace["metadata"]["uid"] = "foreign-namespace".into(); + } else { + locked.objects.get_mut(CONSUMER).unwrap()["spec"]["template"]["metadata"]["annotations"] + [OWNER] = "foreign-registration".into(); + } + } + let before = state.lock().unwrap().objects[CONSUMER].clone(); + migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap(); + let locked = state.lock().unwrap(); + assert_eq!(locked.objects[CONSUMER], before); + assert!(locked.calls.iter().all(|(method, _, _)| method == "GET")); + } +} + +#[tokio::test] +async fn retirement_rechecks_uid_ownership_and_quiescence_before_deleting() { + use std::sync::atomic::{AtomicUsize, Ordering}; + for field in ["uid", "owner", "replicas"] { + let (server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + let value = stopped_owned_consumer(®); + let reads = AtomicUsize::new(0); + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path(CONSUMER)) + .respond_with(move |_: &wiremock::Request| { + let mut current = value.clone(); + if reads.fetch_add(1, Ordering::SeqCst) >= 3 { + match field { + "uid" => current["metadata"]["uid"] = "replacement".into(), + "owner" => { + current["spec"]["template"]["metadata"]["annotations"][OWNER] = + "foreign".into() + } + _ => current["spec"]["replicas"] = 1.into(), + } + } + wiremock::ResponseTemplate::new(200).set_body_json(current) + }) + .with_priority(1) + .mount(&server) + .await; + let error = migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap_err(); + assert!(error.contains("changed after quiescence")); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method != "DELETE") + ); + } +} + +#[tokio::test] +async fn retirement_delete_conflict_is_not_retried_or_forced() { + for code in [403, 409] { + let (server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + wiremock::Mock::given(wiremock::matchers::method("DELETE")) + .and(wiremock::matchers::path(CONSUMER)) + .respond_with(wiremock::ResponseTemplate::new(code).set_body_json(json!({ + "apiVersion":"v1","kind":"Status","status":"Failure", + "reason":if code == 409 {"Conflict"} else {"Forbidden"},"code":code, + "message":"PRIVATE_SENTINEL"}))) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + let error = migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap_err(); + assert_eq!( + error, + format!("Remove stopped owned SRE consumer: Kubernetes status {code}") + ); + assert!(!error.contains("PRIVATE_SENTINEL")); + assert!(state.lock().unwrap().objects.contains_key(CONSUMER)); + } +} + +#[tokio::test] +async fn retirement_waits_for_deletion_without_removing_foreign_finalizers() { + let (_server, client, state) = fixture().await; + let mut reg = registration(); + reg.spec.enabled = false; + seed_stopped_consumer(&state, ®); + { + let mut locked = state.lock().unwrap(); + let deployment = locked.objects.get_mut(CONSUMER).unwrap(); + deployment["metadata"]["deletionTimestamp"] = "2026-09-09T00:00:00Z".into(); + deployment["metadata"]["finalizers"] = json!(["e2e.example/foreign"]); + } + let error = migration::stop_registered_consumer_for_retirement(&client, ®) + .await + .unwrap_err(); + assert!(migration::is_waiting(&error)); + assert!( + state + .lock() + .unwrap() + .calls + .iter() + .all(|(method, _, _)| method == "GET") + ); + assert_eq!( + state.lock().unwrap().objects[CONSUMER]["metadata"]["finalizers"], + json!(["e2e.example/foreign"]) + ); +} + fn reviews(state: &Arc>, kind: &str) -> (KarsSRERegistration, String) { admission_ready(state); let mut locked = state.lock().unwrap(); diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index 83541df77..5a0cbdaea 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -251,6 +251,11 @@ kars sre uninstall --namespace kars-system --release kars ``` Retirement revokes owned private grants and credentials before source cleanup. +The registrar-authorized controller first quiesces and UID/resourceVersion- +fences deletion of its owned SRE Deployment; it does not leave that protected +object for the unprivileged namespace controller. Foreign/replaced consumers +and external finalizers are preserved, not adopted or forced. Retained Retired +records can finish this owned cleanup without reissuing authority. Uninstall/destroy refuse an active enrollment or unretired legacy grants. Retired registrations remain audit records; recreating the source requires explicit enrollment of the new UIDs (`authority stage-source` can atomically From acd34cb16891613c6da6bb4e2dc346e5b35c46e1 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 01:55:56 +0200 Subject: [PATCH 51/62] fix(sre): preserve protected names during collection deletion Real namespace conditions report no remaining content/finalizers but ContentDeletionFailed from no-such-key name in private-identity/role match conditions. Kubernetes 1.31 collection deletion preserves an empty request name while validating each actual oldObject. Resolve optional request/object/oldObject names safely in exactly three match conditions; retain all authorization predicates and Deny bindings byte-identical. Add real ordinary/protected collection DELETE dry-runs proving namespace-controller cleanup is allowed only for ordinary objects and protected targets still require registrar authority. Python: 91 passed; Helm lint and exact guard-diff checks passed. No namespace finalizer force or new privileges. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../templates/sre-authority-admission.yaml | 7 +- .../templates/sre-authority-consumers.yaml | 17 +++- docs/how-to/sre-authority.md | 3 + tests/e2e/sre_authority/bootstrap_probe.py | 8 +- .../sre_authority/collection_delete_probe.py | 87 +++++++++++++++++++ .../collection_delete_probe_test.py | 75 ++++++++++++++++ 6 files changed, 189 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/sre_authority/collection_delete_probe.py create mode 100644 tests/e2e/sre_authority/collection_delete_probe_test.py diff --git a/deploy/helm/kars/templates/sre-authority-admission.yaml b/deploy/helm/kars/templates/sre-authority-admission.yaml index c7b3e1a62..1df203a10 100644 --- a/deploy/helm/kars/templates/sre-authority-admission.yaml +++ b/deploy/helm/kars/templates/sre-authority-admission.yaml @@ -100,9 +100,10 @@ spec: matchConditions: - name: reserved-router-identity expression: >- - request.name == 'sre-api-router' || - (object != null && object.metadata.name == 'sre-api-router') || - (oldObject != null && oldObject.metadata.name == 'sre-api-router') + [request.?name.orValue(''), + object == null ? '' : object.?metadata.?name.orValue(''), + oldObject == null ? '' : oldObject.?metadata.?name.orValue('')] + .exists(n, n == 'sre-api-router') validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml index 6ae135bfc..3e27ac49c 100644 --- a/deploy/helm/kars/templates/sre-authority-consumers.yaml +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -14,7 +14,12 @@ spec: resources: ["deployments", "deployments/scale"] matchConditions: - name: canonical-runtime-consumer - expression: "request.namespace == 'kars-sre' && request.name == 'sre'" + expression: >- + request.?namespace.orValue('') == 'kars-sre' && + [request.?name.orValue(''), + object == null ? '' : object.?metadata.?name.orValue(''), + oldObject == null ? '' : oldObject.?metadata.?name.orValue('')] + .exists(n, n == 'sre') validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') @@ -49,9 +54,13 @@ spec: matchConditions: - name: reserved-sre-role expression: >- - request.name in ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', 'kars-sre-private-diagnostics', - 'kars-sre-registrar', 'kars-sre-retired-agent'] || - (request.namespace == 'kars-sre' && request.name == 'sre-api-self-renew') + [request.?name.orValue(''), + object == null ? '' : object.?metadata.?name.orValue(''), + oldObject == null ? '' : oldObject.?metadata.?name.orValue('')] + .exists(n, + n in ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent'] || + (request.?namespace.orValue('') == 'kars-sre' && n == 'sre-api-self-renew')) validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index 5a0cbdaea..18464a71d 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -256,6 +256,9 @@ fences deletion of its owned SRE Deployment; it does not leave that protected object for the unprivileged namespace controller. Foreign/replaced consumers and external finalizers are preserved, not adopted or forced. Retained Retired records can finish this owned cleanup without reissuing authority. +Name-sensitive admission also inspects the actual object/oldObject name when +collection DELETE omits `request.name`: ordinary collection cleanup remains +valid, while protected items still require registrar authority. Uninstall/destroy refuse an active enrollment or unretired legacy grants. Retired registrations remain audit records; recreating the source requires explicit enrollment of the new UIDs (`authority stage-source` can atomically diff --git a/tests/e2e/sre_authority/bootstrap_probe.py b/tests/e2e/sre_authority/bootstrap_probe.py index 0ee990762..6023668db 100644 --- a/tests/e2e/sre_authority/bootstrap_probe.py +++ b/tests/e2e/sre_authority/bootstrap_probe.py @@ -26,7 +26,8 @@ def failure_site(error): frame = error.__traceback__ while frame: name = Path(frame.tb_frame.f_code.co_filename).name - if name in ("bootstrap_probe.py", "binding_probe.py", "bootstrap_cases.py", "controller_update_probe.py"): + if name in ("bootstrap_probe.py", "binding_probe.py", "bootstrap_cases.py", + "controller_update_probe.py", "collection_delete_probe.py"): result.update(source=name, line=frame.tb_lineno) frame = frame.tb_next return result @@ -258,6 +259,11 @@ def main(root, diagnostics_only, candidate=False, retirement=False): write_report(root, "bootstrap-namespace-cleanup.json", {"cases": cleanup_cases}) if not all(case["matched"] for case in cleanup_cases): raise RuntimeError("Canonical consumer cleanup authority differs from the expected boundary") + from sre_authority.collection_delete_probe import cases as collection_cases + collections = collection_cases(port, policies) + write_report(root, "bootstrap-collection-delete.json", {"cases": collections}) + if not all(case["matched"] for case in collections): + raise RuntimeError("Collection DELETE must preserve ordinary cleanup and protected-object denial") finally: write_report(root, "bootstrap-final.json", collect(port, policies, request)) write_report(root, "bootstrap-controller-stack.json", controller_stack( diff --git a/tests/e2e/sre_authority/collection_delete_probe.py b/tests/e2e/sre_authority/collection_delete_probe.py new file mode 100644 index 000000000..8044ddf35 --- /dev/null +++ b/tests/e2e/sre_authority/collection_delete_probe.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Real nameless collection DELETE checks without removing any guarded fixture.""" + +from urllib.parse import quote + +from .bootstrap_cases import as_tenant +from .bootstrap_diagnostics import api_result +from .common import require +from .registration_schema import request + +NAMESPACE_CONTROLLER = "system:serviceaccount:kube-system:namespace-controller" + + +def cases(port, policies): + fixtures, reports = [], [] + primary_failure = False + definitions = [ + ("v1", "ServiceAccount", "/api/v1/namespaces/kars-sre/serviceaccounts", + "sre-api-router", "kars-sre-private-identity", {"automountServiceAccountToken": False}), + ("rbac.authorization.k8s.io/v1", "Role", "/apis/rbac.authorization.k8s.io/v1/namespaces/kars-sre/roles", + "sre-api-self-renew", "kars-sre-role-authority", {"rules": []}), + ("apps/v1", "Deployment", "/apis/apps/v1/namespaces/kars-sre/deployments", + "sre", "kars-sre-consumer-authority", {"spec": { + "replicas": 0, "selector": {"matchLabels": {"app": "e2e-collection"}}, + "template": {"metadata": {"labels": {"app": "e2e-collection"}}, "spec": { + "automountServiceAccountToken": False, "schedulerName": "kars-e2e-admission-never-schedule", + "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", + "imagePullPolicy": "Never"}]}}}}), + ] + try: + for version, kind, path, protected_name, policy, fields in definitions: + for protected in (False, True): + name = protected_name if protected else "e2e-collection-" + kind.lower() + obj = {"apiVersion": version, "kind": kind, + "metadata": {"name": name, "namespace": "kars-sre"}, **fields} + code, created = request(port, "POST", path, obj) + require(code == 201 and created.get("metadata", {}).get("uid"), + "Collection proof fixture CREATE failed; no adoption allowed") + item_path = path + "/" + name + fixtures.append((item_path, created)) + collection = path + "?fieldSelector=" + quote("metadata.name=" + name, safe="") + for admin in (False, True) if protected else (False,): + code, current = request(port, "GET", item_path) + require(code == 200 and current["metadata"]["uid"] == created["metadata"]["uid"], + "Collection proof fixture identity changed") + options = {"apiVersion": "v1", "kind": "DeleteOptions", "dryRun": ["All"], + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}} + if admin: + code, body = request(port, "DELETE", collection, options) + else: + code, body = as_tenant(port, collection, options, + user=NAMESPACE_CONTROLLER, method="DELETE") + expected = 403 if protected and not admin else 200 + result = api_result(code, body, policies) + result.update({ + "case": f"{kind}-{'protected' if protected else 'ordinary'}-{'admin' if admin else 'namespace-controller'}-collection", + "expectedStatus": expected, + "matched": code == expected and (expected != 403 or ( + policy in result.get("policies", []) + and not set(result.get("categories", [])) & {"evaluation", "no-such-key", "compilation"})), + }) + reports.append(result) + code, after = request(port, "GET", item_path) + require(code == 200 and after["metadata"]["uid"] == created["metadata"]["uid"] + and not after["metadata"].get("deletionTimestamp"), + "Collection DELETE dry-run mutated a fixture") + return reports + except BaseException: + primary_failure = True + raise + finally: + for path, original in reversed(fixtures): + try: + code, current = request(port, "GET", path) + require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"], + "Collection fixture cleanup identity changed") + code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}}) + require(code in (200, 202), "Collection fixture cleanup failed") + except Exception: + if not primary_failure: + raise + print("SRE-DIAG Owned collection fixture cleanup unavailable", flush=True) diff --git a/tests/e2e/sre_authority/collection_delete_probe_test.py b/tests/e2e/sre_authority/collection_delete_probe_test.py new file mode 100644 index 000000000..9c196de66 --- /dev/null +++ b/tests/e2e/sre_authority/collection_delete_probe_test.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import copy +import json +import unittest +from urllib.parse import parse_qs, urlsplit +from unittest.mock import patch + +from sre_authority.collection_delete_probe import NAMESPACE_CONTROLLER, cases + + +class CollectionDeleteProofTests(unittest.TestCase): + def fixture(self): + objects, deletes, probes = {}, [], [] + def api(_port, method, path, obj=None): + parsed = urlsplit(path) + if method == "POST": + value = copy.deepcopy(obj) + value["metadata"].update(uid=value["metadata"]["name"] + "-uid", resourceVersion="1") + objects[path + "/" + value["metadata"]["name"]] = value + return 201, copy.deepcopy(value) + if method == "GET": + return 200, copy.deepcopy(objects[path]) + if parsed.query: + name = parse_qs(parsed.query)["fieldSelector"][0].removeprefix("metadata.name=") + target = parsed.path + "/" + name + self.assertEqual(obj["dryRun"], ["All"]) + else: + target = path + self.assertEqual(obj["preconditions"], { + "uid": objects[target]["metadata"]["uid"], "resourceVersion": objects[target]["metadata"]["resourceVersion"]}) + if not obj.get("dryRun"): + deletes.append(path) + del objects[path] + return 200, {"kind": "Status", "status": "Success"} + def actor(_port, path, obj, **kwargs): + self.assertEqual(kwargs, {"user": NAMESPACE_CONTROLLER, "method": "DELETE"}) + self.assertEqual(obj["dryRun"], ["All"]) + probes.append((path, obj)) + name = parse_qs(urlsplit(path).query)["fieldSelector"][0].removeprefix("metadata.name=") + policy = {"sre-api-router": "kars-sre-private-identity", + "sre-api-self-renew": "kars-sre-role-authority", + "sre": "kars-sre-consumer-authority"}.get(name) + if policy: + return 403, {"kind": "Status", "reason": "Forbidden", "message": policy + " denied do-not-publish"} + return 200, {"kind": "Status", "status": "Success"} + return objects, deletes, probes, api, actor + + def test_collections_are_body_dry_runs_with_specific_guard_denials_and_owned_cleanup(self): + objects, deletes, probes, api, actor = self.fixture() + policies = {name: {} for name in ( + "kars-sre-private-identity", "kars-sre-role-authority", "kars-sre-consumer-authority")} + with patch("sre_authority.collection_delete_probe.request", side_effect=api), \ + patch("sre_authority.collection_delete_probe.as_tenant", side_effect=actor): + results = cases(1, policies) + self.assertEqual(len(results), 9) + self.assertTrue(all(result["matched"] for result in results)) + self.assertEqual(len(probes), 6) + self.assertEqual(len(deletes), 6) + self.assertEqual(objects, {}) + self.assertNotIn("do-not-publish", json.dumps(results)) + + def test_cel_missing_name_errors_are_not_intended_authority_denials(self): + _, _, _, api, _ = self.fixture() + policies = {name: {} for name in ( + "kars-sre-private-identity", "kars-sre-role-authority", "kars-sre-consumer-authority")} + body = {"kind": "Status", "reason": "Forbidden", + "message": "kars-sre-private-identity kars-sre-role-authority kars-sre-consumer-authority " + "evaluation failed: no such key: name do-not-publish"} + with patch("sre_authority.collection_delete_probe.request", side_effect=api), \ + patch("sre_authority.collection_delete_probe.as_tenant", return_value=(403, body)): + results = cases(1, policies) + self.assertFalse(any(result["matched"] for result in results if "namespace-controller" in result["case"])) + self.assertNotIn("do-not-publish", json.dumps(results)) From 5c9be4629e3db62e151f47c2b26addca9491f22e Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 02:09:15 +0200 Subject: [PATCH 52/62] fix(sre): guard collection names with explicit CEL presence checks The API rejected optional-chain name selection at policy creation. Use explicit has-guarded request/object/oldObject names instead, retaining protected oldObject matching and every authorization predicate. Add only bounded compiler field/token/category diagnostics for known public policies; never error bodies. Python 92 passed and Helm lint passed; real API compilation/collection tests remain mandatory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../templates/sre-authority-admission.yaml | 6 ++-- .../templates/sre-authority-consumers.yaml | 16 +++++----- .../sre_authority/bootstrap_diagnostics.py | 32 ++++++++++++++++++- .../e2e/sre_authority/bootstrap_probe_test.py | 14 ++++++++ 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/deploy/helm/kars/templates/sre-authority-admission.yaml b/deploy/helm/kars/templates/sre-authority-admission.yaml index 1df203a10..0c4310fc5 100644 --- a/deploy/helm/kars/templates/sre-authority-admission.yaml +++ b/deploy/helm/kars/templates/sre-authority-admission.yaml @@ -100,9 +100,9 @@ spec: matchConditions: - name: reserved-router-identity expression: >- - [request.?name.orValue(''), - object == null ? '' : object.?metadata.?name.orValue(''), - oldObject == null ? '' : oldObject.?metadata.?name.orValue('')] + [has(request.name) ? request.name : '', + object != null && has(object.metadata) && has(object.metadata.name) ? object.metadata.name : '', + oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) ? oldObject.metadata.name : ''] .exists(n, n == 'sre-api-router') validations: - expression: >- diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml index 3e27ac49c..63c6747a7 100644 --- a/deploy/helm/kars/templates/sre-authority-consumers.yaml +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -15,10 +15,10 @@ spec: matchConditions: - name: canonical-runtime-consumer expression: >- - request.?namespace.orValue('') == 'kars-sre' && - [request.?name.orValue(''), - object == null ? '' : object.?metadata.?name.orValue(''), - oldObject == null ? '' : oldObject.?metadata.?name.orValue('')] + has(request.namespace) && request.namespace == 'kars-sre' && + [has(request.name) ? request.name : '', + object != null && has(object.metadata) && has(object.metadata.name) ? object.metadata.name : '', + oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) ? oldObject.metadata.name : ''] .exists(n, n == 'sre') validations: - expression: >- @@ -54,13 +54,13 @@ spec: matchConditions: - name: reserved-sre-role expression: >- - [request.?name.orValue(''), - object == null ? '' : object.?metadata.?name.orValue(''), - oldObject == null ? '' : oldObject.?metadata.?name.orValue('')] + [has(request.name) ? request.name : '', + object != null && has(object.metadata) && has(object.metadata.name) ? object.metadata.name : '', + oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) ? oldObject.metadata.name : ''] .exists(n, n in ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent'] || - (request.?namespace.orValue('') == 'kars-sre' && n == 'sre-api-self-renew')) + (has(request.namespace) && request.namespace == 'kars-sre' && n == 'sre-api-self-renew')) validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 820a7c221..7951dc464 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -229,8 +229,38 @@ def api_result(code, body, policies): report = {"httpStatus": code} if isinstance(body, dict) and body.get("kind") == "Status": reason = body.get("reason") - report["reason"] = reason if reason in REASONS else "unclassified" + report["reason"] = reason if isinstance(reason, str) and reason in REASONS else "unclassified" report.update(failure_facts(body.get("message"), policies)) + details = body.get("details", {}) + if (code == 422 and reason == "Invalid" and isinstance(details, dict) + and details.get("group") == "admissionregistration.k8s.io" + and details.get("kind") == "ValidatingAdmissionPolicy" + and isinstance(details.get("name"), str) + and details.get("name") in policies): + causes = [] + supplied = details.get("causes", []) + if not isinstance(supplied, list): + return report + for cause in supplied[:32]: + if not isinstance(cause, dict): + continue + field, message = cause.get("field"), cause.get("message") + if not isinstance(field, str) or len(field) > 128 or not re.fullmatch( + r"spec\.(matchConditions|variables|validations)\[\d+\]\.expression", field): + continue + if not isinstance(message, str): + continue + allowed = FIELDS | {"metadata", "request", "object", "oldObject", "orValue", "has", "exists"} + tokens = sorted(set(re.findall( + r"(?:undefined field|undeclared reference to) ['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]", + message)) & allowed) + causes.append({"field": field, "knownTokens": tokens, + "categories": [category for category, needle in [ + ("overload", "matching overload"), ("syntax", "Syntax error"), + ("undefined-field", "undefined field"), ("undeclared-reference", "undeclared reference"), + ("optional", "optional"), ("cost", "cost"), + ] if needle.lower() in message.lower()]}) + report["compilationCauses"] = causes return report diff --git a/tests/e2e/sre_authority/bootstrap_probe_test.py b/tests/e2e/sre_authority/bootstrap_probe_test.py index a9d87de1e..cd18bb1dc 100644 --- a/tests/e2e/sre_authority/bootstrap_probe_test.py +++ b/tests/e2e/sre_authority/bootstrap_probe_test.py @@ -15,6 +15,20 @@ class BootstrapProofTests(unittest.TestCase): + def test_policy_compile_diagnostics_keep_only_known_fields_and_token_categories(self): + body = {"kind": "Status", "reason": "Invalid", "details": { + "group": "admissionregistration.k8s.io", "kind": "ValidatingAdmissionPolicy", + "name": "kars-sre-private-mounts", "causes": [ + {"field": "spec.matchConditions[0].expression", + "message": "compilation failed: undefined field 'metadata'; optional overload do-not-publish"}, + {"field": "spec.arbitrary", "message": "do-not-publish"}, + ]}} + facts = api_result(422, body, POLICIES) + self.assertEqual(facts["compilationCauses"][0]["knownTokens"], ["metadata"]) + self.assertNotIn("do-not-publish", json.dumps(facts)) + body["details"]["name"] = "unrelated" + self.assertNotIn("compilationCauses", api_result(422, body, POLICIES)) + def test_namespace_cleanup_proof_retains_guard_and_fences_all_deletes(self): from sre_authority.bootstrap_cases import namespace_cleanup_cases deployment = {"metadata": {"uid": "owned", "resourceVersion": "2"}} From 633095a8379be377711d422b747a6dcca668c87f Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 02:22:30 +0200 Subject: [PATCH 53/62] test(e2e): retain only public compiler vocabulary in policy failures The API still rejects the collection match condition before execution. Capture its first compiler-description line using only fixed compiler vocabulary and identifiers from the exact public expression; redact all other tokens and retain no arbitrary error body. This is harness-only; all failing technical gates remain enforced. Python: 92 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../sre_authority/bootstrap_diagnostics.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/e2e/sre_authority/bootstrap_diagnostics.py b/tests/e2e/sre_authority/bootstrap_diagnostics.py index 7951dc464..3d615c049 100644 --- a/tests/e2e/sre_authority/bootstrap_diagnostics.py +++ b/tests/e2e/sre_authority/bootstrap_diagnostics.py @@ -245,8 +245,10 @@ def api_result(code, body, policies): if not isinstance(cause, dict): continue field, message = cause.get("field"), cause.get("message") - if not isinstance(field, str) or len(field) > 128 or not re.fullmatch( - r"spec\.(matchConditions|variables|validations)\[\d+\]\.expression", field): + location = re.fullmatch( + r"spec\.(matchConditions|variables|validations)\[(\d+)\]\.expression", field + ) if isinstance(field, str) and len(field) <= 128 else None + if not location: continue if not isinstance(message, str): continue @@ -254,12 +256,31 @@ def api_result(code, body, policies): tokens = sorted(set(re.findall( r"(?:undefined field|undeclared reference to) ['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]", message)) & allowed) - causes.append({"field": field, "knownTokens": tokens, + entry = {"field": field, "knownTokens": tokens, "categories": [category for category, needle in [ ("overload", "matching overload"), ("syntax", "Syntax error"), ("undefined-field", "undefined field"), ("undeclared-reference", "undeclared reference"), ("optional", "optional"), ("cost", "cost"), - ] if needle.lower() in message.lower()]}) + ] if needle.lower() in message.lower()]} + definitions = policies[details["name"]].get("spec", {}).get(location[1], []) + index = int(location[2]) + expression = definitions[index].get("expression", "") if index < len(definitions) else "" + vocabulary = allowed | set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", expression)) | { + "error", "input", "expression", "must", "evaluate", "evaluates", "return", "returns", + "type", "bool", "boolean", "string", "int", "list", "map", "dyn", "invalid", "argument", + "macro", "expected", "found", "no", "matching", "overload", "applied", "to", "in", + "not", "a", "an", "is", "of", "undeclared", "reference", "undefined", "field", + "mismatched", "extraneous", "syntax", "token", "reserved", "identifier", "unsupported", + "supported", "allowed", "size", "exceeds", "maximum", "limit", "cost", "compilation", + } + headline = message.partition("compilation failed:")[2].splitlines() + if headline: + entry["compilerDescription"] = re.sub( + r"[A-Za-z0-9_]+", + lambda match: match[0] if match[0] in vocabulary or match[0].lower() in vocabulary else "[redacted]", + re.sub(r"[\x00-\x1f\x7f]", "?", headline[0].strip())[:512], + ) + causes.append(entry) report["compilationCauses"] = causes return report From 3f20fac439960fe5144ca69b355bd1dc3167a465 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 02:32:02 +0200 Subject: [PATCH 54/62] fix(sre): avoid mixed string and dyn lists in collection guards Actual CEL compiler evidence reports expected string but found dyn for the temporary list of request/object names. Replace that list/comprehension with direct presence-guarded OR comparisons, retaining each protected request/object/oldObject name and every registrar authorization condition. No coercion, collection bypass, new privileges or finalizer changes. Python 92 and Helm lint pass; actual API compile/collection cases remain mandatory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../templates/sre-authority-admission.yaml | 9 +++--- .../templates/sre-authority-consumers.yaml | 29 ++++++++++++------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/deploy/helm/kars/templates/sre-authority-admission.yaml b/deploy/helm/kars/templates/sre-authority-admission.yaml index 0c4310fc5..a480448d2 100644 --- a/deploy/helm/kars/templates/sre-authority-admission.yaml +++ b/deploy/helm/kars/templates/sre-authority-admission.yaml @@ -100,10 +100,11 @@ spec: matchConditions: - name: reserved-router-identity expression: >- - [has(request.name) ? request.name : '', - object != null && has(object.metadata) && has(object.metadata.name) ? object.metadata.name : '', - oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) ? oldObject.metadata.name : ''] - .exists(n, n == 'sre-api-router') + (has(request.name) && request.name == 'sre-api-router') || + (object != null && has(object.metadata) && has(object.metadata.name) && + object.metadata.name == 'sre-api-router') || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && + oldObject.metadata.name == 'sre-api-router') validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml index 63c6747a7..a6e43f981 100644 --- a/deploy/helm/kars/templates/sre-authority-consumers.yaml +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -16,10 +16,9 @@ spec: - name: canonical-runtime-consumer expression: >- has(request.namespace) && request.namespace == 'kars-sre' && - [has(request.name) ? request.name : '', - object != null && has(object.metadata) && has(object.metadata.name) ? object.metadata.name : '', - oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) ? oldObject.metadata.name : ''] - .exists(n, n == 'sre') + ((has(request.name) && request.name == 'sre') || + (object != null && has(object.metadata) && has(object.metadata.name) && object.metadata.name == 'sre') || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && oldObject.metadata.name == 'sre')) validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') @@ -54,13 +53,21 @@ spec: matchConditions: - name: reserved-sre-role expression: >- - [has(request.name) ? request.name : '', - object != null && has(object.metadata) && has(object.metadata.name) ? object.metadata.name : '', - oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) ? oldObject.metadata.name : ''] - .exists(n, - n in ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', - 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent'] || - (has(request.namespace) && request.namespace == 'kars-sre' && n == 'sre-api-self-renew')) + (has(request.name) && request.name in + ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent']) || + (object != null && has(object.metadata) && has(object.metadata.name) && object.metadata.name in + ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent']) || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && oldObject.metadata.name in + ['kars-sre-reader', 'kars-sre-action-author', 'kars-sre-router-renew', + 'kars-sre-private-diagnostics', 'kars-sre-registrar', 'kars-sre-retired-agent']) || + (has(request.namespace) && request.namespace == 'kars-sre' && + ((has(request.name) && request.name == 'sre-api-self-renew') || + (object != null && has(object.metadata) && has(object.metadata.name) && + object.metadata.name == 'sre-api-self-renew') || + (oldObject != null && has(oldObject.metadata) && has(oldObject.metadata.name) && + oldObject.metadata.name == 'sre-api-self-renew'))) validations: - expression: >- authorizer.group('kars.azure.com').resource('karssreregistrations') From 6ec85420b7ba211801b3aa0ac0e9de538791497c Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 03:25:57 +0200 Subject: [PATCH 55/62] test(e2e): verify the explicitly configured Hermes image pin Full Kind now completes legacy and fresh SRE acceptance, including namespace cleanup; its sole remaining failure is the ordinary Hermes image-name check conflicting with the explicit SRE stand-in pin. Verify the exact controller-configured HERMES_RUNTIME_IMAGE plus actual Hermes dispatch instead of requiring a substring. Keep the unconfigured fallback and reject wrong/missing images or BYO dispatch; do not blanket-accept stand-ins or skip the assertion. Python: 93 passed; shell syntax checks passed. No production change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/run.sh | 22 ++++++++++++++++------ tests/e2e/sre-authority.sh | 10 ++++++++++ tests/e2e/sre_authority/harness_test.py | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 00b97055b..8923b9af9 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -1341,8 +1341,8 @@ EOF test_runtime_hermes() { # KarsSandbox of kind Hermes should be processed by the controller: # plan_hermes dispatches, namespace is created, and the agent - # container's image carries the hermes runtime tag (kars-runtime-hermes) - # rather than the OpenClaw default. Mirrors test_runtime_anthropic and + # container uses the configured Hermes image (including the SRE fixture's + # explicit stand-in pin), with actual Hermes runtime dispatch. Mirrors test_runtime_anthropic and # follows the same tolerance pattern: Deployment may not materialize # if there's no real InferencePolicy provider in this lane, so the # image-tag assertion is diag-only when no Deployment is present. @@ -1372,14 +1372,24 @@ EOF echo "" fail "Hermes runtime: no namespace" fi - local image + local image configured runtime_kind image=$(kubectl get deploy -n kars-e2e-hermes e2e-hermes -o jsonpath='{.spec.template.spec.containers[?(@.name=="agent")].image}' 2>/dev/null || true) if [ -n "$image" ]; then - if echo "$image" | grep -qE "hermes|kars-runtime-hermes"; then - pass "Hermes Deployment uses hermes runtime image ($image)" + if ! configured=$(kubectl get deployment kars-controller -n kars-system \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="controller")].env[?(@.name=="HERMES_RUNTIME_IMAGE")].value}'); then + fail "Hermes runtime: could not inspect the configured image" + return + fi + if ! runtime_kind=$(kubectl get deployment e2e-hermes -n kars-e2e-hermes \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="agent")].env[?(@.name=="KARS_RUNTIME_KIND")].value}'); then + fail "Hermes runtime: could not inspect actual runtime dispatch" + return + fi + if sre_hermes_image_matches "$image" "$configured" "$runtime_kind"; then + pass "Hermes Deployment preserves the configured image and Hermes runtime dispatch ($image)" else echo " [diag] container image: $image" - fail "Hermes Deployment image does not reference hermes runtime" + fail "Hermes Deployment does not match its configured image or runtime dispatch" fi else echo " [diag] no Deployment yet (likely no InferencePolicy provider in this lane)" diff --git a/tests/e2e/sre-authority.sh b/tests/e2e/sre-authority.sh index 41aa3cac4..db4e005c9 100644 --- a/tests/e2e/sre-authority.sh +++ b/tests/e2e/sre-authority.sh @@ -11,6 +11,16 @@ sre_migration_helm_wait_arg() { esac } +sre_hermes_image_matches() { + local image="$1" configured="$2" runtime="$3" + [ "$runtime" = "Hermes" ] && [ -n "$image" ] || return 1 + if [ -n "$configured" ]; then + [ "$image" = "$configured" ] + else + [[ "$image" == *hermes* ]] + fi +} + sre_authority_phase() { local phase="$1" output result=0 line info "SRE authority acceptance: ${phase}" diff --git a/tests/e2e/sre_authority/harness_test.py b/tests/e2e/sre_authority/harness_test.py index 8d79ecb4d..a45c66113 100644 --- a/tests/e2e/sre_authority/harness_test.py +++ b/tests/e2e/sre_authority/harness_test.py @@ -34,6 +34,25 @@ def json(self): class HarnessTests(unittest.TestCase): + def test_hermes_runtime_assertion_verifies_exact_pin_and_runtime_without_blanket_standin_acceptance(self): + helper = Path(__file__).resolve().parents[1] / "sre-authority.sh" + cases = [ + ("kars-sandbox-e2e:dev", "kars-sandbox-e2e:dev", "Hermes", 0), + ("wrong:latest", "kars-sandbox-e2e:dev", "Hermes", 1), + ("kars-sandbox-e2e:dev", "", "Hermes", 1), + ("custom/runtime:latest", "custom/runtime:latest", "Hermes", 0), + ("kars-runtime-hermes:latest", "", "Hermes", 0), + ("kars-runtime-hermes:latest", "", "BYO", 1), + ("", "", "Hermes", 1), + ] + for image, configured, runtime, expected in cases: + result = subprocess.run( + ["bash", "-c", 'source "$1"; sre_hermes_image_matches "$2" "$3" "$4"', + "hermes-image-test", str(helper), image, configured, runtime], + capture_output=True, text=True, timeout=5, check=False) + with self.subTest(image=image, configured=configured, runtime=runtime): + self.assertEqual(result.returncode, expected) + def test_log_reader_diagnostics_never_echo_log_text_or_error_body(self): root = Path(__file__).resolve().parents[3] facts = log_reader_facts(root, {"error": "406 Not Acceptable", From 203e2322ad22512f0889e1f512ed36ac278b5a42 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 04:16:26 +0200 Subject: [PATCH 56/62] test(e2e): fence collection previews against status-only RV races Full Kind on 6ec85420 passes 161 tests with complete legacy/fresh SRE retirement and namespace cleanup. The isolated API proof exposed a deployment-status RV race, not an authorization failure. Keep UID/RV preconditions on every dry-run and retry at most twice only after confirming an actual RV change and no content/identity change beyond status/managedFields. Never retry authorization failures, unchanged-RV conflicts, production writes or cleanup writes. Pause only the zero-replica collection fixture to avoid unrelated rollout bookkeeping. Python: 95 passed; existing focused CLI SRE tests: 11 passed with unchanged timeout limits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../sre_authority/collection_delete_probe.py | 46 +++++++++++++++---- .../collection_delete_probe_test.py | 36 ++++++++++++++- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/tests/e2e/sre_authority/collection_delete_probe.py b/tests/e2e/sre_authority/collection_delete_probe.py index 8044ddf35..c8875709b 100644 --- a/tests/e2e/sre_authority/collection_delete_probe.py +++ b/tests/e2e/sre_authority/collection_delete_probe.py @@ -3,6 +3,8 @@ """Real nameless collection DELETE checks without removing any guarded fixture.""" +import copy +import time from urllib.parse import quote from .bootstrap_cases import as_tenant @@ -13,6 +15,37 @@ NAMESPACE_CONTROLLER = "system:serviceaccount:kube-system:namespace-controller" +def stable_object(value): + value = copy.deepcopy(value) + value.pop("status", None) + value.get("metadata", {}).pop("resourceVersion", None) + value.get("metadata", {}).pop("managedFields", None) + return value + + +def dry_run_delete(port, collection, item_path, current, admin): + for attempt in range(3): + options = {"apiVersion": "v1", "kind": "DeleteOptions", "dryRun": ["All"], + "preconditions": {"uid": current["metadata"]["uid"], + "resourceVersion": current["metadata"]["resourceVersion"]}} + if admin: + code, body = request(port, "DELETE", collection, options) + else: + code, body = as_tenant(port, collection, options, + user=NAMESPACE_CONTROLLER, method="DELETE") + if (code != 409 or not isinstance(body, dict) or body.get("reason") != "Conflict" + or attempt == 2): + return code, body + status, refreshed = request(port, "GET", item_path) + require(status == 200 and stable_object(refreshed) == stable_object(current), + "Collection dry-run target changed beyond controller status; no retry permitted") + if refreshed["metadata"]["resourceVersion"] == current["metadata"]["resourceVersion"]: + return code, body + current = refreshed + time.sleep(0.2) + raise AssertionError("Collection dry-run exceeded its fixed attempt bound") + + def cases(port, policies): fixtures, reports = [], [] primary_failure = False @@ -23,7 +56,7 @@ def cases(port, policies): "sre-api-self-renew", "kars-sre-role-authority", {"rules": []}), ("apps/v1", "Deployment", "/apis/apps/v1/namespaces/kars-sre/deployments", "sre", "kars-sre-consumer-authority", {"spec": { - "replicas": 0, "selector": {"matchLabels": {"app": "e2e-collection"}}, + "replicas": 0, "paused": True, "selector": {"matchLabels": {"app": "e2e-collection"}}, "template": {"metadata": {"labels": {"app": "e2e-collection"}}, "spec": { "automountServiceAccountToken": False, "schedulerName": "kars-e2e-admission-never-schedule", "containers": [{"name": "probe", "image": "registry.invalid/kars-admission-proof:never", @@ -45,14 +78,7 @@ def cases(port, policies): code, current = request(port, "GET", item_path) require(code == 200 and current["metadata"]["uid"] == created["metadata"]["uid"], "Collection proof fixture identity changed") - options = {"apiVersion": "v1", "kind": "DeleteOptions", "dryRun": ["All"], - "preconditions": {"uid": current["metadata"]["uid"], - "resourceVersion": current["metadata"]["resourceVersion"]}} - if admin: - code, body = request(port, "DELETE", collection, options) - else: - code, body = as_tenant(port, collection, options, - user=NAMESPACE_CONTROLLER, method="DELETE") + code, body = dry_run_delete(port, collection, item_path, current, admin) expected = 403 if protected and not admin else 200 result = api_result(code, body, policies) result.update({ @@ -77,6 +103,8 @@ def cases(port, policies): code, current = request(port, "GET", path) require(code == 200 and current["metadata"]["uid"] == original["metadata"]["uid"], "Collection fixture cleanup identity changed") + require(stable_object(current) == stable_object(original), + "Collection fixture content changed; cleanup preserved it") code, _ = request(port, "DELETE", path, {"apiVersion": "v1", "kind": "DeleteOptions", "preconditions": {"uid": current["metadata"]["uid"], "resourceVersion": current["metadata"]["resourceVersion"]}}) diff --git a/tests/e2e/sre_authority/collection_delete_probe_test.py b/tests/e2e/sre_authority/collection_delete_probe_test.py index 9c196de66..1e4a173f9 100644 --- a/tests/e2e/sre_authority/collection_delete_probe_test.py +++ b/tests/e2e/sre_authority/collection_delete_probe_test.py @@ -7,10 +7,44 @@ from urllib.parse import parse_qs, urlsplit from unittest.mock import patch -from sre_authority.collection_delete_probe import NAMESPACE_CONTROLLER, cases +from sre_authority.collection_delete_probe import NAMESPACE_CONTROLLER, cases, dry_run_delete class CollectionDeleteProofTests(unittest.TestCase): + def test_dry_run_retries_only_status_only_rv_conflicts_and_keeps_both_preconditions(self): + original = {"metadata": {"uid": "owned", "resourceVersion": "1"}, "spec": {"replicas": 0}} + updated = copy.deepcopy(original) + updated["metadata"]["resourceVersion"] = "2" + updated["status"] = {"observedGeneration": 1} + with patch("sre_authority.collection_delete_probe.request", return_value=(200, updated)), \ + patch("sre_authority.collection_delete_probe.as_tenant", + side_effect=[(409, {"reason": "Conflict"}), (200, {"kind": "Status"})]) as actor, \ + patch("sre_authority.collection_delete_probe.time.sleep"): + code, _ = dry_run_delete(1, "/collection?fieldSelector=name", "/item", original, False) + self.assertEqual(code, 200) + self.assertEqual([call.args[2]["preconditions"] for call in actor.call_args_list], [ + {"uid": "owned", "resourceVersion": "1"}, {"uid": "owned", "resourceVersion": "2"}]) + self.assertTrue(all(call.args[2]["dryRun"] == ["All"] for call in actor.call_args_list)) + self.assertEqual(original["metadata"]["resourceVersion"], "1") + + def test_dry_run_does_not_retry_replacement_content_change_or_authorization_failure(self): + original = {"metadata": {"uid": "owned", "resourceVersion": "1"}, "spec": {"replicas": 0}} + for refreshed in ( + {"metadata": {"uid": "other", "resourceVersion": "2"}, "spec": {"replicas": 0}}, + {"metadata": {"uid": "owned", "resourceVersion": "2"}, "spec": {"replicas": 1}}, + ): + with patch("sre_authority.collection_delete_probe.request", return_value=(200, refreshed)), \ + patch("sre_authority.collection_delete_probe.as_tenant", + return_value=(409, {"reason": "Conflict"})) as actor, self.assertRaises(AssertionError): + dry_run_delete(1, "/collection", "/item", original, False) + actor.assert_called_once() + with patch("sre_authority.collection_delete_probe.request") as read, \ + patch("sre_authority.collection_delete_probe.as_tenant", + return_value=(403, {"reason": "Forbidden"})) as actor: + self.assertEqual(dry_run_delete(1, "/collection", "/item", original, False)[0], 403) + actor.assert_called_once() + read.assert_not_called() + def fixture(self): objects, deletes, probes = {}, [], [] def api(_port, method, path, obj=None): From 141657f9ed5ec2b30575d7e04503318e7e8a4057 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 11:33:50 +0200 Subject: [PATCH 57/62] fix(sre): stage compatible APIs and bind Azure teardown to every target cluster Use the Helm 3/4 built-in waiter before explicit enrollment, and repair only the exact historical action params schema with UID/resourceVersion fences before dependent policies. Preserve mandatory migration and policy readiness gates. Bind resource-group deletion to a pinned subscription, complete AKS resourceUID inventory and ARM-issued private kubeconfigs. Reject context mismatches and --all --local; never modify global kubeconfig. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/commands/destroy.test.ts | 49 ++++ cli/src/commands/destroy.ts | 45 ++-- cli/src/lib/azure-destroy-target.test.ts | 224 ++++++++++++++++++ cli/src/lib/azure-destroy-target.ts | 147 ++++++++++++ cli/src/lib/sre-action-crd.ts | 99 ++++++++ cli/src/lib/sre-authority.test.ts | 11 +- cli/src/lib/sre-helm.test.ts | 29 ++- cli/src/lib/sre-helm.ts | 16 +- cli/src/lib/sre-stage.test.ts | 205 ++++++++++++++++ cli/src/lib/sre-stage.ts | 35 ++- docs/how-to/sre-authority.md | 42 ++++ tests/e2e/sre_authority/fixtures.py | 12 + tests/e2e/sre_authority/migration.py | 28 +++ .../staging_compatibility_test.py | 84 +++++++ 14 files changed, 991 insertions(+), 35 deletions(-) create mode 100644 cli/src/commands/destroy.test.ts create mode 100644 cli/src/lib/azure-destroy-target.test.ts create mode 100644 cli/src/lib/azure-destroy-target.ts create mode 100644 cli/src/lib/sre-action-crd.ts create mode 100644 cli/src/lib/sre-stage.test.ts create mode 100644 tests/e2e/sre_authority/staging_compatibility_test.py diff --git a/cli/src/commands/destroy.test.ts b/cli/src/commands/destroy.test.ts new file mode 100644 index 000000000..67e698af1 --- /dev/null +++ b/cli/src/commands/destroy.test.ts @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { destroyCommand } from "./destroy.js"; + +const { execute, target, guard } = vi.hoisted(() => ({ + execute: vi.fn(), target: vi.fn(), guard: vi.fn(), +})); +vi.mock("execa", () => ({ execa: execute })); +vi.mock("../lib/azure-destroy-target.js", () => ({ withAzureDestroyTarget: target })); +vi.mock("../lib/sre-authority.js", () => ({ assertDestroySafe: guard })); +vi.mock("ora", () => ({ default: () => ({ start() { return this; }, succeed: vi.fn(), fail: vi.fn() }) })); +afterEach(() => { vi.restoreAllMocks(); vi.clearAllMocks(); }); + +describe("destroy --all cannot bypass target-bound retirement", () => { + it.each([["--local"], ["--local", "--cloud"], ["sre", "--local"], ["anything", "--local"]].map(flags => ({ flags })))( + "rejects Azure --all with local flags: $flags", async ({ flags }) => { + await expect(destroyCommand().parseAsync(["node", "destroy", "--all", "--yes", ...flags])).rejects.toThrow("cannot be combined"); + expect(execute).not.toHaveBeenCalled(); + expect(target).not.toHaveBeenCalled(); + }, + ); + it.each([[], ["sre"], ["unrelated"], ["--cloud"]].map(flags => ({ flags })))("always binds Azure deletion for $flags", async ({ flags }) => { + const azure = vi.fn().mockResolvedValue({ stdout: "" }); + target.mockImplementation(async (_exec, _rg, _sub, _context, destroy) => destroy(azure)); + await destroyCommand().parseAsync(["node", "destroy", "--all", "--yes", "--resource-group", "B", + "--subscription", "subscription-id", "--context", "A", ...flags]); + expect(target).toHaveBeenCalledWith(execute, "B", "subscription-id", "A", expect.any(Function)); + expect(guard).not.toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + expect(azure.mock.calls.map(([, args]) => args.slice(0, 2))).toEqual([ + ["group", "delete"], ["cognitiveservices", "account"], ["keyvault", "purge"], + ]); + }); + it("never starts deletion when target proof fails", async () => { + target.mockRejectedValue(new Error("target mismatch")); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation(() => { throw new Error("exit 1"); }); + await expect(destroyCommand().parseAsync(["node", "destroy", "--all", "--yes"])).rejects.toThrow("exit 1"); + expect(execute).not.toHaveBeenCalled(); + }); + it("does not fetch credentials or mutate anything merely to show --all confirmation", async () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + await destroyCommand().parseAsync(["node", "destroy", "--all"]); + expect(target).not.toHaveBeenCalled(); + expect(execute).not.toHaveBeenCalled(); + }); +}); diff --git a/cli/src/commands/destroy.ts b/cli/src/commands/destroy.ts index 40ffd1822..decf6420b 100644 --- a/cli/src/commands/destroy.ts +++ b/cli/src/commands/destroy.ts @@ -5,6 +5,7 @@ import { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; import { assertDestroySafe } from "../lib/sre-authority.js"; +import { withAzureDestroyTarget } from "../lib/azure-destroy-target.js"; export function destroyCommand(): Command { const cmd = new Command("destroy"); @@ -17,13 +18,15 @@ export function destroyCommand(): Command { .option("--cloud", "Destroy AKS cloud sandbox only (skip Docker)", false) .option("--all", "Destroy ALL resources (AKS, ACR, KV, AOAI — deletes the resource group)", false) .option("-g, --resource-group ", "Resource group name") + .option("--subscription ", "Azure subscription for --all (defaults to the selected Azure account)") .option("--region ", "Azure region (used to derive resource group)", "eastus2") - .option("--context ", "Kubernetes context to use (defaults to current)") + .option("--context ", "Kubernetes context (current for sandbox teardown; verified against Azure target for --all)") .action(async (name: string | undefined, options) => { const rg = options.resourceGroup || `kars-${options.region}`; // Propagate --context to every kubectl invocation in this command. const kctlCtx = options.context ? ["--context", options.context] : []; - if ((!options.local || options.cloud) && (!name || name === "sre" || options.all)) { + if (options.all && options.local) throw new Error("--all deletes Azure resources and cannot be combined with --local"); + if (!options.all && (!options.local || options.cloud) && (!name || name === "sre")) { const { execa } = await import("execa"); await assertDestroySafe((file,args,commandOptions) => execa(file,[...kctlCtx,...args],commandOptions)); @@ -47,25 +50,27 @@ export function destroyCommand(): Command { const { execa } = await import("execa"); const baseName = "kars"; - // Delete the resource group (async) - await execa("az", [ - "group", "delete", "--name", rg, "--yes", "--no-wait", "--output", "none", - ], { stdio: "pipe" }); - - // Purge soft-deleted resources so a fresh 'up' works without conflicts - spinner.text = "Purging soft-deleted Azure OpenAI account..."; - await execa("az", [ - "cognitiveservices", "account", "purge", - "--name", `${baseName}-aoai`, - "--resource-group", rg, - "--location", options.region, - "--output", "none", - ], { stdio: "pipe" }).catch(() => {}); + await withAzureDestroyTarget(execa, rg, options.subscription, options.context, async azure => { + // Delete the resource group (async) + await azure("az", [ + "group", "delete", "--name", rg, "--yes", "--no-wait", "--output", "none", + ], { stdio: "pipe" }); - spinner.text = "Purging soft-deleted Key Vault..."; - await execa("az", [ - "keyvault", "purge", "--name", `${baseName}-kv`, - ], { stdio: "pipe" }).catch(() => {}); + // Purge soft-deleted resources so a fresh 'up' works without conflicts + spinner.text = "Purging soft-deleted Azure OpenAI account..."; + await azure("az", [ + "cognitiveservices", "account", "purge", + "--name", `${baseName}-aoai`, + "--resource-group", rg, + "--location", options.region, + "--output", "none", + ], { stdio: "pipe" }).catch(() => {}); + + spinner.text = "Purging soft-deleted Key Vault..."; + await azure("az", [ + "keyvault", "purge", "--name", `${baseName}-kv`, + ], { stdio: "pipe" }).catch(() => {}); + }); spinner.succeed(`Resource group '${rg}' deletion initiated + soft-deleted resources purged`); } catch (error) { diff --git a/cli/src/lib/azure-destroy-target.test.ts b/cli/src/lib/azure-destroy-target.test.ts new file mode 100644 index 000000000..051a769e5 --- /dev/null +++ b/cli/src/lib/azure-destroy-target.test.ts @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { existsSync, readFileSync, statSync } from "node:fs"; +import { dirname } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { withAzureDestroyTarget } from "./azure-destroy-target.js"; +import type { Execute } from "./sre-authority.js"; + +const subscription = "11111111-1111-1111-1111-111111111111"; +const otherSubscription = "22222222-2222-2222-2222-222222222222"; +const group = "target-B"; +const groupId = `/subscriptions/${subscription}/resourceGroups/${group}`; +const ca = readFileSync(new URL("../../../a2a-gateway/testdata/test-cert.pem", import.meta.url)); +const endpoint = { server: "https://aks.example.test/", "certificate-authority-data": ca.toString("base64") }; +const metadata = (name: string) => ({ name, uid: name, resourceVersion: "1" }); + +function cluster(name: string) { + return { id: `${groupId}/providers/Microsoft.ContainerService/managedClusters/${name}`, + name, resourceGroup: group, resourceUid: `${name}-immutable-resource-uid` }; +} +function credentials(name: string) { + return JSON.stringify({ apiVersion: "v1", kind: "Config", "current-context": name, + contexts: [{ name, context: { cluster: name, user: name } }], + clusters: [{ name, cluster: endpoint }], users: [{ name, user: { token: "synthetic-test-token" } }] }); +} +function fixture(names = ["cluster-b"]) { + const state = { + clusters: names.map(cluster), inventoryCalls: 0, active: new Set(), unavailable: "", + selectedUid: "cluster-b-kube-system", selectedEndpoint: { ...endpoint }, + changedInventory: undefined as undefined | ReturnType[], + }; + const files = new Set(); + const execute = vi.fn(async (file, args) => { + if (file === "az") { + if (args[0] === "account") return { stdout: subscription }; + expect(args.slice(-2)).toEqual(["--subscription", subscription]); + if (args[0] === "group" && args[1] === "show") return { stdout: groupId }; + if (args[0] === "aks" && args[1] === "list") { + state.inventoryCalls++; + return { stdout: JSON.stringify(state.inventoryCalls > 1 ? state.changedInventory ?? state.clusters : state.clusters) }; + } + if (args[0] === "rest") { + expect(args.slice(0, 3)).toEqual(["rest", "--method", "post"]); + const url = args[args.indexOf("--url") + 1]; + const target = state.clusters.find(cluster => + url === `${cluster.id.toLowerCase()}/listClusterUserCredential?api-version=2024-10-01`); + expect(target).toBeDefined(); + const name = target!.name; + if (state.unavailable === name) throw new Error("AKS credentials Forbidden"); + return { stdout: JSON.stringify({ kubeconfigs: [{ name: "clusterUser", value: Buffer.from(credentials(name)).toString("base64") }] }) }; + } + if (args[0] === "group" && args[1] === "delete") return { stdout: "" }; + } + if (file === "kubectl") { + const context = args[args.indexOf("--context") + 1]; + if (args.includes("config")) return { stdout: JSON.stringify(state.selectedEndpoint) }; + if (args.includes("--kubeconfig")) { + const path = args[args.indexOf("--kubeconfig") + 1]; + files.add(path); + expect(statSync(path).mode & 0o777).toBe(0o600); + expect(statSync(dirname(path)).mode & 0o777).toBe(0o700); + expect(JSON.parse(readFileSync(path, "utf8"))["current-context"]).toBe(context); + } + const start = args.indexOf("get"); + const [kind, name] = args.slice(start + 1, start + 3); + if (kind === "namespace") return { stdout: name === "kube-system" ? JSON.stringify({ + metadata: { ...metadata(name), uid: context === "selected" ? state.selectedUid : `${context}-kube-system` }, + }) : "" }; + if (kind === "crd") return { stdout: state.active.has(context) ? JSON.stringify({ metadata: metadata(name) }) : "" }; + if (kind === "karssreregistrations.kars.azure.com") return { stdout: JSON.stringify({ + metadata: { ...metadata("canonical"), generation: 1 }, spec: { enabled: true }, + status: { phase: "Ready", observedGeneration: 1 }, + }) }; + } + throw new Error(`Unexpected fixture command: ${file} ${args.join(" ")}`); + }); + const remove = vi.fn(async (azure: Execute) => { + await azure("az", ["group", "delete", "--name", group, "--yes", "--no-wait", "--output", "none"], { stdio: "pipe" }); + }); + const run = (context?: string, exec: Execute = execute, sub?: string) => withAzureDestroyTarget(exec, group, sub, context, remove); + const cleaned = () => { + for (const path of files) { + expect(existsSync(path)).toBe(false); + expect(existsSync(dirname(path))).toBe(false); + } + }; + return { state, execute, files, remove, run, cleaned }; +} + +describe("Azure teardown retirement target binding", () => { + it("checks ALL resource-group AKS APIs, ignores global current-context and pins subscription through deletion", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + await f.run(); + expect(f.remove).toHaveBeenCalledOnce(); + expect(f.state.inventoryCalls).toBe(2); + expect(f.files.size).toBe(2); + expect(f.execute.mock.calls.some(([file, args]) => file === "kubectl" && !args.includes("--kubeconfig"))).toBe(false); + expect(f.execute.mock.calls.some(([, args]) => args.includes("current-context") || args.includes("--overwrite-existing"))).toBe(false); + expect(f.execute.mock.calls.some(([, args]) => args.includes("get-credentials") || args.includes("convert-kubeconfig"))).toBe(false); + const deletion = f.execute.mock.calls.find(([, args]) => args[0] === "group" && args[1] === "delete")!; + expect(deletion[1].slice(-2)).toEqual(["--subscription", subscription]); + f.cleaned(); + }); + + it("resolves an explicit subscription once and never follows a later selected-account switch", async () => { + const f = fixture(); + await f.run(undefined, f.execute, subscription); + expect(f.execute.mock.calls[0][1]).toEqual(["account", "show", "--subscription", subscription, "--query", "id", "--output", "tsv"]); + expect(f.execute.mock.calls.filter(([, args]) => args[0] === "account")).toHaveLength(1); + f.cleaned(); + }); + + it("accepts an explicit context only with the ARM credential TLS endpoint AND real cluster UID", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + await f.run("selected"); + expect(f.remove).toHaveBeenCalledOnce(); + expect(f.files.size).toBe(2); + f.cleaned(); + }); + + it.each(["uid", "ca", "server"])("rejects context A versus deletion target B (%s mismatch)", async field => { + const f = fixture(); + if (field === "uid") f.state.selectedUid = "cluster-a-kube-system"; + if (field === "ca") f.state.selectedEndpoint["certificate-authority-data"] = Buffer.concat([ca, Buffer.from("\n")]).toString("base64"); + if (field === "server") f.state.selectedEndpoint.server = "https://different.example.test/"; + await expect(f.run("selected")).rejects.toThrow("--context does not match"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it("blocks an active registration in a second cluster even when the selected context is retired/empty", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + f.state.active.add("cluster-c"); + await expect(f.run("selected")).rejects.toThrow("Retire"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it("fails closed when any cluster's credentials/API are inaccessible", async () => { + const f = fixture(["cluster-b", "cluster-c"]); + f.state.unavailable = "cluster-c"; + await expect(f.run()).rejects.toThrow("Cannot obtain AKS user credentials"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it.each(["new-cluster", "removed-cluster", "replaced-cluster"])("blocks a %s inventory race before deletion", async change => { + const f = fixture(); + f.state.changedInventory = change === "new-cluster" ? [...f.state.clusters, cluster("new")] + : change === "removed-cluster" ? [] : [{ ...f.state.clusters[0], resourceUid: "replacement-uid" }]; + await expect(f.run()).rejects.toThrow("inventory changed"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it.each(["id", "name", "resourceUid", "resourceGroup"])("rejects ambiguous or foreign ARM inventory %s", async key => { + const f = fixture(); + (f.state.clusters[0] as any)[key] = key === "id" ? f.state.clusters[0].id.replace(subscription, otherSubscription) : ""; + await expect(f.run()).rejects.toThrow("exact ARM/resourceUid identity"); + expect(f.remove).not.toHaveBeenCalled(); + }); + + it("rejects duplicate clusters rather than treating repeated names as distinct proof", async () => { + const f = fixture(); + f.state.clusters.push({ ...f.state.clusters[0] }); + await expect(f.run()).rejects.toThrow("exact ARM/resourceUid identity"); + expect(f.remove).not.toHaveBeenCalled(); + }); + + it.each(["empty", "duplicate", "bad-base64", "bad-yaml", "insecure-tls"])( + "rejects %s ARM credentials without leaking their response", async kind => { + const f = fixture(); + const marker = "credential-response-must-not-be-printed"; + let value: any = { kubeconfigs: [] }; + if (kind === "duplicate") value.kubeconfigs = [{ value: marker }, { value: marker }]; + if (kind === "bad-base64") value.kubeconfigs = [{ value: marker }]; + if (kind === "bad-yaml") value.kubeconfigs = [{ value: Buffer.from(`secret: [${marker}`).toString("base64") }]; + if (kind === "insecure-tls") { + const config = JSON.parse(credentials("cluster-b")); + config.clusters[0].cluster["insecure-skip-tls-verify"] = true; + value.kubeconfigs = [{ value: Buffer.from(JSON.stringify(config)).toString("base64") }]; + } + const execute: Execute = (file, args, options) => args[0] === "rest" + ? Promise.resolve({ stdout: JSON.stringify(value) }) : f.execute(file, args, options); + let error: unknown; + try { await f.run(undefined, execute); } catch (caught) { error = caught; } + expect(error).toBeInstanceOf(Error); + expect(String(error)).not.toContain(marker); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }, + ); + + it.each(["account", "group", "list", "api"])("does not interpret a %s failure as cluster absence", async point => { + const f = fixture(); + const execute: Execute = (file, args, options) => (point === "account" && args[0] === "account") + || (point === "group" && args[0] === "group" && args[1] === "show") + || (point === "list" && args[1] === "list") || (point === "api" && file === "kubectl") + ? Promise.reject(new Error("Forbidden target proof")) : f.execute(file, args, options); + await expect(f.run(undefined, execute)).rejects.toThrow("Forbidden target proof"); + expect(f.remove).not.toHaveBeenCalled(); + f.cleaned(); + }); + + it("permits an authoritatively empty AKS inventory, but cannot claim an explicit unrelated context", async () => { + const f = fixture([]); + await f.run(); + expect(f.remove).toHaveBeenCalledOnce(); + expect(f.state.inventoryCalls).toBe(2); + f.remove.mockClear(); + await expect(f.run("selected")).rejects.toThrow("--context does not match"); + expect(f.remove).not.toHaveBeenCalled(); + }); + + it("rejects subscription retargeting even inside the destructive callback", async () => { + const f = fixture(); + await expect(withAzureDestroyTarget(f.execute, group, undefined, undefined, async azure => { + await azure("az", ["group", "delete", "--name", group, "--subscription", otherSubscription], { stdio: "pipe" }); + })).rejects.toThrow("not the deployment subscription"); + expect(f.execute.mock.calls.some(([, args]) => args[1] === "delete")).toBe(false); + f.cleaned(); + }); +}); diff --git a/cli/src/lib/azure-destroy-target.ts b/cli/src/lib/azure-destroy-target.ts new file mode 100644 index 000000000..0452a2c63 --- /dev/null +++ b/cli/src/lib/azure-destroy-target.ts @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID, X509Certificate } from "node:crypto"; +import { mkdir, open, unlink, rmdir } from "node:fs/promises"; +import { resolve } from "node:path"; +import { parse } from "yaml"; +import { pinAzureSubscription } from "./azure-subscription.js"; +import { assertDestroySafe, get, type Execute } from "./sre-authority.js"; + +interface Cluster { id: string; name: string; resourceGroup: string; resourceUid: string } + +function trustedEndpoint(cluster: any): string { + if (!cluster || typeof cluster.server !== "string" + || typeof cluster["certificate-authority-data"] !== "string" + || cluster["insecure-skip-tls-verify"] || cluster["proxy-url"] || cluster["tls-server-name"] + || cluster["certificate-authority"]) throw new Error("AKS target lacks an unambiguous TLS-authenticated API endpoint"); + const url = new URL(cluster.server); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash || url.pathname !== "/") { + throw new Error("Invalid AKS API endpoint"); + } + const ca = Buffer.from(cluster["certificate-authority-data"], "base64"); + new X509Certificate(ca); + return `${url.href}\n${createHash("sha256").update(ca).digest("hex")}`; +} + +function credentialIdentity(stdout: string): { context: string; endpoint: string } { + let config; + try { config = parse(stdout); } catch { + throw new Error("Azure returned malformed AKS credentials; deletion is blocked"); + } + const context = config?.["current-context"]; + if (typeof context !== "string" || !context || !Array.isArray(config.contexts) || !Array.isArray(config.clusters)) { + throw new Error("Azure returned an invalid AKS kubeconfig"); + } + const contexts = config.contexts.filter((item: any) => item.name === context); + if (contexts.length !== 1) throw new Error("Azure returned an ambiguous AKS context"); + const clusters = config.clusters.filter((item: any) => item.name === contexts[0].context?.cluster); + if (clusters.length !== 1) throw new Error("Azure returned an ambiguous AKS cluster"); + return { context, endpoint: trustedEndpoint(clusters[0].cluster) }; +} + +async function userCredentials(azure: Execute, id: string): Promise { + // Some az aks get-credentials versions run kubelogin conversion even with + // --file -, potentially touching the global kubeconfig. Read ARM directly. + let result; + try { + const response = await azure("az", ["rest", "--method", "post", "--url", + `${id}/listClusterUserCredential?api-version=2024-10-01`, "--output", "json"], { stdio: "pipe" }); + result = JSON.parse(response.stdout); + } catch { + // Command/YAML errors must never print credential-bearing response bodies. + throw new Error("Cannot obtain AKS user credentials from the selected ARM resource; deletion is blocked"); + } + const configs = result?.kubeconfigs; + if (!Array.isArray(configs) || configs.length !== 1 || typeof configs[0]?.value !== "string" + || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(configs[0].value) + || !configs[0].value) throw new Error("Azure returned ambiguous or invalid AKS credentials; deletion is blocked"); + return Buffer.from(configs[0].value, "base64").toString("utf8"); +} + +/** Bind every retirement check to ARM-issued credentials for every AKS resource + * in the deletion target. Neither current-context nor an AKS name is proof. + * The callback receives the same subscription pin used by the entire preflight. */ +export async function withAzureDestroyTarget( + execute: Execute, resourceGroup: string, requestedSubscription: string | undefined, + requestedContext: string | undefined, destroy: (azure: Execute) => Promise, +): Promise { + const subscription = (await execute("az", ["account", "show", + ...(requestedSubscription ? ["--subscription", requestedSubscription] : []), + "--query", "id", "--output", "tsv"], { stdio: "pipe" })).stdout.trim(); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(subscription)) { + throw new Error("Azure did not resolve a subscription ID; resource-group deletion is blocked"); + } + const azure: Execute = (file, args, options) => { + if (file !== "az") throw new Error("Azure teardown executor cannot use an unbound Kubernetes context"); + return execute(file, pinAzureSubscription(args, subscription), options); + }; + const groupId = `/subscriptions/${subscription}/resourceGroups/${resourceGroup}`; + async function inventory(): Promise { + const group = (await azure("az", ["group", "show", "--name", resourceGroup, + "--query", "id", "--output", "tsv"], { stdio: "pipe" })).stdout.trim(); + if (group.toLowerCase() !== groupId.toLowerCase()) throw new Error("Azure resource-group target identity mismatch"); + const result: unknown = JSON.parse((await azure("az", ["aks", "list", "--resource-group", resourceGroup, + "--output", "json"], { stdio: "pipe" })).stdout); + if (!Array.isArray(result)) throw new Error("Azure AKS inventory is malformed; deletion is blocked"); + const seen = new Set(); + const clusters: Cluster[] = []; + for (const cluster of result) { + if (!cluster || typeof cluster.name !== "string" || !cluster.name + || typeof cluster.id !== "string" || typeof cluster.resourceGroup !== "string" + || typeof cluster.resourceUid !== "string" || !cluster.resourceUid.trim() + || cluster.resourceGroup.toLowerCase() !== resourceGroup.toLowerCase() + || cluster.id.toLowerCase() !== `${groupId}/providers/Microsoft.ContainerService/managedClusters/${cluster.name}`.toLowerCase() + || seen.has(cluster.id.toLowerCase())) { + throw new Error("AKS inventory lacks exact ARM/resourceUid identity; update Azure CLI or review the target before deletion"); + } + seen.add(cluster.id.toLowerCase()); + clusters.push({ id: cluster.id.toLowerCase(), name: cluster.name, resourceGroup: cluster.resourceGroup, + resourceUid: cluster.resourceUid }); + } + return clusters.sort((a, b) => a.id.localeCompare(b.id)); + } + const clusters = await inventory(); + let selected: { endpoint: string; uid: string } | undefined; + if (requestedContext) { + const endpoint = trustedEndpoint(JSON.parse((await execute("kubectl", ["--context", requestedContext, + "config", "view", "--minify", "--raw", "-o", "jsonpath={.clusters[0].cluster}"], { stdio: "pipe" })).stdout)); + const ns = await get((file, args, options) => + execute(file, ["--context", requestedContext, ...args], options), "namespace", "kube-system"); + if (!ns) throw new Error("Selected context lacks a live kube-system identity"); + selected = { endpoint, uid: ns.metadata.uid! }; + } + let contextMatched = !selected; + const directory = resolve(`.kars-destroy-${randomUUID()}`); + await mkdir(directory, { mode: 0o700 }); + const owned: string[] = []; + try { + for (const [index, cluster] of clusters.entries()) { + const credentials = await userCredentials(azure, cluster.id); + const identity = credentialIdentity(credentials); + const path = resolve(directory, `cluster-${index}.yaml`); + const file = await open(path, "wx", 0o600); + owned.push(path); + try { await file.writeFile(credentials); } finally { await file.close(); } + const bound: Execute = (file, args, options) => { + if (file !== "kubectl") throw new Error("Retirement preflight requires the bound AKS API"); + return execute(file, ["--kubeconfig", path, "--context", identity.context, ...args], options); + }; + const before = await get(bound, "namespace", "kube-system"); + if (!before) throw new Error("AKS API lacks its live kube-system identity"); + if (selected?.endpoint === identity.endpoint && selected.uid === before.metadata.uid) contextMatched = true; + await assertDestroySafe(bound); + const after = await get(bound, "namespace", "kube-system"); + if (after?.metadata.uid !== before.metadata.uid) throw new Error("AKS API identity changed during retirement preflight"); + } + if (!contextMatched) throw new Error("--context does not match any TLS/UID-verified AKS cluster in the selected resource group"); + if (JSON.stringify(await inventory()) !== JSON.stringify(clusters)) { + throw new Error("Azure AKS inventory changed during retirement preflight; deletion is blocked"); + } + await destroy(azure); + } finally { + for (const path of owned) await unlink(path); + // No recursive deletion: foreign files are never removed. + await rmdir(directory); + } +} diff --git a/cli/src/lib/sre-action-crd.ts b/cli/src/lib/sre-action-crd.ts new file mode 100644 index 000000000..73fd402bd --- /dev/null +++ b/cli/src/lib/sre-action-crd.ts @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { get, type ApiObject, type Execute } from "./sre-authority.js"; + +export const ACTION_CRD = "karssreactions.kars.azure.com"; +const PARAMS = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/action/properties/params"; +// Complete immutable baseline spec, not a name/label-based adoption rule. +// Azure/kars 8b206065608593667a40665b3f48225ef9ce278d (BASE365). +const BASELINE = "2e119f9dee47426bdf1332e346a5f75f8b6a2d0d6c2a2781903406e7e25b577e"; + +function canonical(value: any): string { + const sort = (v: any): any => Array.isArray(v) ? v.map(sort) + : v && typeof v === "object" ? Object.fromEntries(Object.keys(v).sort().map(k => [k, sort(v[k])])) : v; + return JSON.stringify(sort(value)); +} + +function normalizedSpec(object: ApiObject): any { + const spec = structuredClone(object.spec); + if (!spec?.names || !Array.isArray(spec.versions)) throw new Error("Action CRD schema is missing"); + // Only API defaulting is ignored; custom fields, versions and validations + // are never overwritten or treated as the recognized historical schema. + spec.names.categories ??= []; + if (spec.names.listKind === "KarsSREActionList") delete spec.names.listKind; + if (canonical(spec.conversion) === '{"strategy":"None"}') delete spec.conversion; + if (spec.preserveUnknownFields === false) delete spec.preserveUnknownFields; + for (const version of spec.versions) { + if (version.deprecated === false) delete version.deprecated; + for (const column of version.additionalPrinterColumns ?? []) { + if (column.priority === 0) delete column.priority; + } + } + return spec; +} + +export async function planActionCrd( + execute: Execute, desired: ApiObject, namespace: string, release: string, helm: boolean, +): Promise<() => Promise> { + const spec = normalizedSpec(desired); + const legacy = structuredClone(spec); + const params = legacy.versions[0]?.schema?.openAPIV3Schema?.properties?.spec?.properties?.action?.properties?.params; + if (desired.kind !== "CustomResourceDefinition" || desired.metadata.name !== ACTION_CRD + || params?.["x-kubernetes-preserve-unknown-fields"] !== true || "additionalProperties" in params) { + throw new Error("Staged chart lacks the compatible action CRD params repair"); + } + delete params["x-kubernetes-preserve-unknown-fields"]; + params.additionalProperties = true; + if (createHash("sha256").update(canonical(legacy)).digest("hex") !== BASELINE) { + throw new Error("Unrecognized action CRD chart schema; explicitly review compatibility before staging policies"); + } + const existing = await get(execute, "customresourcedefinition", ACTION_CRD); + if (!existing) { + return async () => { + await execute("kubectl", ["create", "-f", "-"], { stdio: "pipe", input: JSON.stringify({ + ...desired, metadata: { ...desired.metadata, ...(helm ? { + labels: { ...desired.metadata.labels, "app.kubernetes.io/managed-by": "Helm" }, + annotations: { "meta.helm.sh/release-name": release, "meta.helm.sh/release-namespace": namespace }, + } : {}) }, + }) }); + await established(execute); + }; + } + const annotations = existing.metadata.annotations ?? {}; + const manager = existing.metadata.labels?.["app.kubernetes.io/managed-by"]; + if (existing.metadata.deletionTimestamp || existing.metadata.ownerReferences?.length + || (manager && manager !== "Helm") + || (annotations["meta.helm.sh/release-name"] && (!helm || annotations["meta.helm.sh/release-name"] !== release)) + || (annotations["meta.helm.sh/release-namespace"] && (!helm || annotations["meta.helm.sh/release-namespace"] !== namespace)) + || (annotations["kars.azure.com/sre-authority-staged"] && annotations["kars.azure.com/sre-authority-staged"] !== namespace) + || (annotations["kars.azure.com/sre-authority-release"] && annotations["kars.azure.com/sre-authority-release"] !== release) + || (manager === "Helm" && (!helm || annotations["meta.helm.sh/release-name"] !== release + || annotations["meta.helm.sh/release-namespace"] !== namespace))) { + throw new Error("Foreign or terminating action CRD; review its installation ownership before staging"); + } + const current = normalizedSpec(existing); + const repair = canonical(current) === canonical(legacy); + if (!repair && canonical(current) !== canonical(spec)) { + throw new Error("Customized action CRD schema; explicitly review the params compatibility repair before staging policies"); + } + return async () => { + // Even the already-repaired case tests the complete reviewed snapshot before + // installing dependent policies. No legacy ownership annotation is adopted. + await execute("kubectl", ["patch", "customresourcedefinition", ACTION_CRD, "--type=json", "-p", JSON.stringify([ + { op: "test", path: "/metadata/uid", value: existing.metadata.uid }, + { op: "test", path: "/metadata/resourceVersion", value: existing.metadata.resourceVersion }, + ...(repair ? [ + { op: "test", path: `${PARAMS}/additionalProperties`, value: true }, + { op: "remove", path: `${PARAMS}/additionalProperties` }, + { op: "add", path: `${PARAMS}/x-kubernetes-preserve-unknown-fields`, value: true }, + ] : []), + ])], { stdio: "pipe" }); + await established(execute); + }; +} + +async function established(execute: Execute): Promise { + await execute("kubectl", ["wait", "--for=condition=Established", `crd/${ACTION_CRD}`, "--timeout=60s"], { stdio: "pipe" }); +} diff --git a/cli/src/lib/sre-authority.test.ts b/cli/src/lib/sre-authority.test.ts index 9b34835bd..90db36a16 100644 --- a/cli/src/lib/sre-authority.test.ts +++ b/cli/src/lib/sre-authority.test.ts @@ -5,6 +5,9 @@ import { describe,expect,it,vi } from "vitest"; import { assertDestroySafe,assertRollbackSafe,assertSafeMutation,enroll,preview,waitForAuthority,type Execute } from "./sre-authority.js"; import { stageSource } from "./sre-source.js"; import { stageAuthority } from "./sre-stage.js"; +import { readFileSync } from "node:fs"; + +const actionCrd=readFileSync(new URL("../../../deploy/helm/kars/templates/crd-karssreaction.yaml",import.meta.url),"utf8"); function fixture() { const objects:Record={ @@ -216,6 +219,7 @@ describe("SRE cluster registrar boundary",()=>{ return {stdout:'[{"name":"kars","namespace":"kars-system","status":"pending-upgrade"}]'}; } if(file==="helm"&&args[0]==="version")return {stdout:version}; + if(file==="helm"&&args[0]==="template")return {stdout:actionCrd}; if(file==="helm"&&args[0]==="upgrade")return {stdout:""}; return f.execute(file,args,options); }); @@ -223,8 +227,9 @@ describe("SRE cluster registrar boundary",()=>{ const upgrade=execute.mock.calls.find(([file,args])=>file==="helm"&&args[0]==="upgrade"); expect(upgrade?.[1]).toContain("--reset-then-reuse-values"); expect(upgrade?.[1]).toContain("sre.authorityStage=true"); - expect(upgrade?.[1]).toContain(dryRun?"--dry-run=server":"--wait"); - expect(execute.mock.calls.some(([,args])=>["install","template","create","patch"].includes(args[0]))).toBe(false); + expect(upgrade?.[1]).toContain(dryRun?"--dry-run=server":version.startsWith("v4.")?"--wait=legacy":"--wait"); + expect(execute.mock.calls.some(([,args])=>args[0]==="install")).toBe(false); + expect(execute.mock.calls.some(([,args])=>args[0]==="create")).toBe(!dryRun); } finally { warn.mockRestore(); } }); @@ -237,7 +242,7 @@ describe("SRE cluster registrar boundary",()=>{ const execute=vi.fn(async(file,args,options)=>{ if(file==="helm"&&args[0]==="list")return {stdout:"[]"}; if(file==="helm")return {stdout:JSON.stringify({kind:"CustomResourceDefinition", - apiVersion:"apiextensions.k8s.io/v1",metadata:{name:"karssreregistrations.kars.azure.com"},spec:{scope:"Cluster"}})}; + apiVersion:"apiextensions.k8s.io/v1",metadata:{name:"karssreregistrations.kars.azure.com"},spec:{scope:"Cluster"}})+"\n"+actionCrd}; return f.execute(file,args,options); }); delete f.objects["crd//karssreregistrations.kars.azure.com"]; diff --git a/cli/src/lib/sre-helm.test.ts b/cli/src/lib/sre-helm.test.ts index b3bc8f513..b5336cf6b 100644 --- a/cli/src/lib/sre-helm.test.ts +++ b/cli/src/lib/sre-helm.test.ts @@ -4,7 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { execa } from "execa"; import type { Execute } from "./sre-authority.js"; -import { listSreHelmReleases } from "./sre-helm.js"; +import { listSreHelmReleases, sreHelmStageWait } from "./sre-helm.js"; const inventory = JSON.stringify([{ name: "kars", namespace: "workspace", status: "pending-upgrade" }]); const unsupported = () => Object.assign(new Error("Helm flag rejected"), { @@ -66,7 +66,7 @@ describe("SRE Helm release inventory compatibility", () => { "does not assume default all-status semantics for %j", async version => { const execute = vi.fn().mockRejectedValueOnce(unsupported()) .mockResolvedValueOnce({ stdout: version }); - await expect(listSreHelmReleases(execute, "workspace")).rejects.toThrow("only Helm 4"); + await expect(listSreHelmReleases(execute, "workspace")).rejects.toThrow(/only Helm 4|Unsupported Helm version/); expect(execute).toHaveBeenCalledTimes(2); }, ); @@ -79,4 +79,29 @@ describe("SRE Helm release inventory compatibility", () => { await expect(listSreHelmReleases(execute, "workspace")).rejects.toBe(denied); expect(execute).toHaveBeenCalledTimes(3); }); + + describe("SRE staging waits only for built-ins before explicit enrollment", () => { + it.each([["v3.16.4", "--wait"], ["v4.2.4", "--wait=legacy"], ["v4.0.0-rc.1", "--wait=legacy"]])( + "selects %s's bounded built-in waiter", async (version, expected) => { + const execute = vi.fn().mockResolvedValue({ stdout: version }); + expect(await sreHelmStageWait(execute)).toBe(expected); + expect(execute.mock.calls).toEqual([ + ["helm", ["version", "--template", "{{.Version}}"], { stdio: "pipe" }], + ]); + }, + ); + it.each(["", "v5.0.0", "v4.2", "v3.16.4\nwarning", "unknown"])("fails closed for %j", async version => { + await expect(sreHelmStageWait(vi.fn().mockResolvedValue({ stdout: version }))) + .rejects.toThrow("Unsupported Helm"); + }); + it("does not replace version errors with a default waiter", async () => { + const error = new Error("Helm unavailable"); + await expect(sreHelmStageWait(vi.fn().mockRejectedValue(error))).rejects.toBe(error); + }); + it("passes the real installed Helm upgrade parser without a Kubernetes connection", async () => { + const wait = await sreHelmStageWait(execa); + const { stdout } = await execa("helm", ["upgrade", "kars", "chart", wait, "--timeout", "8m", "--help"], { stdio: "pipe" }); + expect(stdout).toContain("helm upgrade"); + }); + }); }); diff --git a/cli/src/lib/sre-helm.ts b/cli/src/lib/sre-helm.ts index e885a73fe..d83e5a2ce 100644 --- a/cli/src/lib/sre-helm.ts +++ b/cli/src/lib/sre-helm.ts @@ -3,6 +3,19 @@ import type { Execute } from "./sre-authority.js"; +export async function sreHelmMajor(execute: Execute): Promise<3 | 4> { + const { stdout } = await execute("helm", ["version", "--template", "{{.Version}}"], { stdio: "pipe" }); + const match = /^v([34])\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?$/.exec(stdout.trim()); + if (!match) throw new Error("Unsupported Helm version; SRE staging requires Helm 3 or 4."); + return Number(match[1]) as 3 | 4; +} + +export async function sreHelmStageWait(execute: Execute): Promise { + // The Helm 4 watcher waits for custom-resource Ready, which requires the + // explicit enrollment AFTER staging. Its legacy waiter checks built-ins/APIs. + return await sreHelmMajor(execute) === 4 ? "--wait=legacy" : "--wait"; +} + function removedAllFlag(error: unknown): boolean { return error instanceof Error && "exitCode" in error && error.exitCode === 1 @@ -15,8 +28,7 @@ export async function listSreHelmReleases(execute: Execute, namespace: string): return (await execute("helm", ["list", "-n", namespace, "--all", "-o", "json"], { stdio: "pipe" })).stdout; } catch (error) { if (!removedAllFlag(error)) throw error; - const { stdout } = await execute("helm", ["version", "--template", "{{.Version}}"], { stdio: "pipe" }); - if (!/^v4\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?$/.test(stdout.trim())) { + if (await sreHelmMajor(execute) !== 4) { throw new Error("Unsupported Helm release inventory: only Helm 4 can omit the --all flag.", { cause: error }); } // Helm 3 needs --all; Helm 4 removed it and lists every status by default. diff --git a/cli/src/lib/sre-stage.test.ts b/cli/src/lib/sre-stage.test.ts new file mode 100644 index 000000000..e2890070d --- /dev/null +++ b/cli/src/lib/sre-stage.test.ts @@ -0,0 +1,205 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { execa } from "execa"; +import { parse } from "yaml"; +import { describe, expect, it, vi } from "vitest"; +import { ACTION_CRD, planActionCrd } from "./sre-action-crd.js"; +import { stageAuthority } from "./sre-stage.js"; +import type { Execute } from "./sre-authority.js"; + +const action = parse(readFileSync(new URL("../../../deploy/helm/kars/templates/crd-karssreaction.yaml", import.meta.url), "utf8")); +const registration = { apiVersion: "apiextensions.k8s.io/v1", kind: "CustomResourceDefinition", + metadata: { name: "karssreregistrations.kars.azure.com" }, spec: { scope: "Cluster" } }; +const policy = { apiVersion: "admissionregistration.k8s.io/v1", kind: "ValidatingAdmissionPolicy", + metadata: { name: "kars-sre-test" }, spec: { failurePolicy: "Fail" } }; +const params = (object: any) => object.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.action.properties.params; + +function fixture(helm = false) { + const existing = structuredClone(action); + existing.metadata.uid = "action-uid"; + existing.metadata.resourceVersion = "17"; + existing.metadata.annotations = { "operator.example/keep": "custom metadata" }; + if (helm) { + existing.metadata.labels["app.kubernetes.io/managed-by"] = "Helm"; + Object.assign(existing.metadata.annotations, { "meta.helm.sh/release-name": "kars", "meta.helm.sh/release-namespace": "kars-system" }); + } + delete params(existing)["x-kubernetes-preserve-unknown-fields"]; + params(existing).additionalProperties = true; + const controller = { metadata: { name: "kars-controller", uid: "controller-uid", resourceVersion: "2" }, + spec: { template: { spec: { serviceAccountName: "kars-controller", containers: [{ name: "controller", image: "old:latest" }] } } } }; + const execute = vi.fn(async (file, args, options) => { + if (file === "helm") { + if (args[0] === "list") return { stdout: helm ? '[{"name":"kars","namespace":"kars-system"}]' : "[]" }; + if (args[0] === "version") return { stdout: "v4.2.4" }; + if (args[0] === "template") return { stdout: [action, registration, policy].map(obj => JSON.stringify(obj)).join("\n---\n") }; + if (args[0] === "upgrade") return { stdout: "" }; + } + if (args[0] === "auth") return { stdout: "yes" }; + if (args[0] === "get") return { stdout: args[2] === ACTION_CRD ? JSON.stringify(existing) + : args[1] === "deployment" ? JSON.stringify(controller) : "" }; + if (args[0] === "patch" && args[2] === ACTION_CRD) { + const operations = JSON.parse(args[args.indexOf("-p") + 1]); + for (const operation of operations) { + const segments = operation.path.slice(1).split("/"); + const target = segments.slice(0, -1).reduce((obj: any, key: string) => obj[key], existing); + const key = segments.at(-1); + if (operation.op === "test" && JSON.stringify(target[key]) !== JSON.stringify(operation.value)) throw new Error("409 UID/RV conflict"); + if (operation.op === "remove") delete target[key]; + if (operation.op === "add") target[key] = operation.value; + } + } + if (args[0] === "create") JSON.parse(options.input!); + return { stdout: "" }; + }); + const run = (dry = false, exec: Execute = execute) => stageAuthority(exec, "chart", "kars-system", "kars", "controller:latest", "router:latest", dry); + return { existing, controller, execute, run }; +} + +describe("existing action API prerequisite compatibility", () => { + it("feeds the actual offline Helm render through template staging with the repaired action API first", async () => { + const f = fixture(); + const execute: Execute = (file, args, options) => file === "helm" && args[0] === "template" + ? execa(file, args, options) : f.execute(file, args, options); + await stageAuthority(execute, fileURLToPath(new URL("../../../deploy/helm/kars", import.meta.url)), + "kars-system", "kars", "controller:latest", "router:latest", false); + expect(params(f.existing)["x-kubernetes-preserve-unknown-fields"]).toBe(true); + const created = f.execute.mock.calls.filter(([, args]) => args[0] === "create").map(([, , options]) => JSON.parse(options.input!)); + expect(created.some(obj => obj.metadata.name === registration.metadata.name)).toBe(true); + expect(created.filter(obj => obj.kind === "ValidatingAdmissionPolicy")).toHaveLength(14); + }); + + it.each([false, true])("repairs ONLY recognized legacy params before dependent policies (Helm: %s)", async helm => { + const f = fixture(helm); + const before = structuredClone(f.existing); + await f.run(); + const repaired = structuredClone(before); + delete params(repaired).additionalProperties; + params(repaired)["x-kubernetes-preserve-unknown-fields"] = true; + expect(f.existing).toEqual(repaired); + const calls = f.execute.mock.calls; + const patch = calls.findIndex(([, args]) => args[0] === "patch" && args[2] === ACTION_CRD); + const wait = calls.findIndex(([, args]) => args[0] === "wait" && args.includes(`crd/${ACTION_CRD}`)); + const dependent = calls.findIndex(([file, args, options]) => helm ? file === "helm" && args[0] === "upgrade" + : args[0] === "create" && JSON.parse(options.input!).kind === "ValidatingAdmissionPolicy"); + expect(patch).toBeGreaterThan(0); + expect(wait).toBeGreaterThan(patch); + expect(dependent).toBeGreaterThan(wait); + const operations = JSON.parse(calls[patch][1].at(-1)!); + expect(operations.slice(0, 2)).toEqual([ + { op: "test", path: "/metadata/uid", value: "action-uid" }, + { op: "test", path: "/metadata/resourceVersion", value: "17" }, + ]); + expect(f.existing.metadata.annotations["kars.azure.com/sre-authority-staged"]).toBeUndefined(); + if (helm) expect(calls[dependent][1]).toEqual(expect.arrayContaining(["--wait=legacy", "--timeout", "8m"])); + }); + + it("recognizes only Kubernetes API defaults while preserving their serialized fields", async () => { + const f = fixture(); + f.existing.spec.names.listKind = "KarsSREActionList"; + delete f.existing.spec.names.categories; + f.existing.spec.conversion = { strategy: "None" }; + f.existing.spec.preserveUnknownFields = false; + f.existing.spec.versions[0].deprecated = false; + f.existing.spec.versions[0].additionalPrinterColumns[0].priority = 0; + await f.run(); + expect(f.existing.spec.conversion).toEqual({ strategy: "None" }); + expect(f.existing.spec.names.listKind).toBe("KarsSREActionList"); + expect(f.existing.spec.names.categories).toBeUndefined(); + }); + + it.each([ + (obj: any) => { obj.metadata.annotations["meta.helm.sh/release-name"] = "foreign"; }, + (obj: any) => { obj.metadata.annotations["meta.helm.sh/release-namespace"] = "foreign"; }, + (obj: any) => { obj.metadata.annotations["kars.azure.com/sre-authority-staged"] = "foreign"; }, + (obj: any) => { obj.metadata.annotations["kars.azure.com/sre-authority-release"] = "foreign"; }, + (obj: any) => { obj.metadata.labels["app.kubernetes.io/managed-by"] = "other-operator"; }, + (obj: any) => { obj.metadata.ownerReferences = [{ uid: "foreign-owner" }]; }, + (obj: any) => { obj.metadata.deletionTimestamp = "2026-09-10T00:00:00Z"; }, + (obj: any) => { params(obj).properties = { custom: { type: "string" } }; }, + (obj: any) => { obj.spec.versions[0].schema.openAPIV3Schema.properties.spec.required.push("diagnosis"); }, + (obj: any) => { obj.spec.versions.push(structuredClone(obj.spec.versions[0])); }, + (obj: any) => { obj.spec.conversion = { strategy: "Webhook" }; }, + ])("refuses foreign/customized schemas without writing any authority resource: %#", async modify => { + const f = fixture(); + modify(f.existing); + await expect(f.run()).rejects.toThrow(/Foreign|Customized/); + expect(f.execute.mock.calls.some(([, args]) => ["patch", "create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it.each(["uid", "resourceVersion"])("rejects a racing %s replacement before policy creation", async field => { + const f = fixture(); + const stage = await planActionCrd(f.execute, action, "kars-system", "kars", false); + f.existing.metadata[field] = "replaced"; + await expect(stage()).rejects.toThrow("409"); + expect(params(f.existing).additionalProperties).toBe(true); + }); + + it.each([false, true])("propagates prerequisite failure before policies or Helm upgrade (Helm: %s)", async helm => { + const f = fixture(helm); + const execute = vi.fn(async (file, args, options) => { + if (args[0] === "wait" && args.includes(`crd/${ACTION_CRD}`)) throw new Error("Established timeout"); + return f.execute(file, args, options); + }); + + await expect(f.run(false, execute)).rejects.toThrow("Established timeout"); + expect(execute.mock.calls.some(([, args]) => ["create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it.each([false, true])("never continues after a forbidden prerequisite PATCH (Helm: %s)", async helm => { + const f = fixture(helm); + const execute: Execute = (file, args, options) => args[0] === "patch" && args[2] === ACTION_CRD + ? Promise.reject(new Error("Forbidden action API update")) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("Forbidden action API update"); + expect(f.execute.mock.calls.some(([, args]) => ["create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it("requires API establishment even when an existing registration schema is unchanged", async () => { + const f = fixture(); + const execute = vi.fn(async (file, args, options) => { + if (args[0] === "get" && args[2] === registration.metadata.name) return { stdout: JSON.stringify({ + ...registration, metadata: { ...registration.metadata, uid: "registration-api", resourceVersion: "1" }, + }) }; + if (args[0] === "wait" && args.includes(`crd/${registration.metadata.name}`)) throw new Error("Registration API not established"); + return f.execute(file, args, options); + }); + await expect(f.run(false, execute)).rejects.toThrow("Registration API not established"); + expect(execute.mock.calls.some(([, args]) => args[0] === "create")).toBe(false); + }); + + it("performs all template ownership preflights before repairing the prerequisite", async () => { + const f = fixture(); + const execute: Execute = (file, args, options) => args[0] === "get" && args[2] === policy.metadata.name + ? Promise.resolve({ stdout: JSON.stringify({ ...policy, metadata: { ...policy.metadata, uid: "foreign", resourceVersion: "1" }, + spec: { failurePolicy: "Ignore" } }) }) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("Unowned"); + expect(f.execute.mock.calls.some(([, args]) => args[0] === "patch")).toBe(false); + }); + + it.each([false, true])("dry-run never repairs APIs or mutates the controller (Helm: %s)", async helm => { + const f = fixture(helm); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + await f.run(true); + expect(params(f.existing).additionalProperties).toBe(true); + expect(f.execute.mock.calls.some(([, args]) => ["patch", "create", "wait", "rollout"].includes(args[0]))).toBe(false); + } finally { log.mockRestore(); } + }); + + it.each(["v5.0.0", "invalid"])("rejects unsupported Helm %s before any prerequisite write", async version => { + const f = fixture(true); + const execute: Execute = (file, args, options) => args[0] === "version" + ? Promise.resolve({ stdout: version }) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("Unsupported Helm"); + expect(f.execute.mock.calls.some(([, args]) => ["patch", "create", "upgrade"].includes(args[0]))).toBe(false); + }); + + it("retains fatal rollout failures instead of accepting an unready controller", async () => { + const f = fixture(); + const execute: Execute = (file, args, options) => args[0] === "rollout" + ? Promise.reject(new Error("controller rollout failed")) : f.execute(file, args, options); + await expect(f.run(false, execute)).rejects.toThrow("controller rollout failed"); + }); +}); diff --git a/cli/src/lib/sre-stage.ts b/cli/src/lib/sre-stage.ts index e1e77063d..c460b4820 100644 --- a/cli/src/lib/sre-stage.ts +++ b/cli/src/lib/sre-stage.ts @@ -3,7 +3,8 @@ import { parseAllDocuments } from "yaml"; import { get, requireRegistrar, type ApiObject, type Execute } from "./sre-authority.js"; -import { listSreHelmReleases } from "./sre-helm.js"; +import { listSreHelmReleases, sreHelmStageWait } from "./sre-helm.js"; +import { ACTION_CRD, planActionCrd } from "./sre-action-crd.js"; function parts(image: string): [string,string] { const index=image.lastIndexOf(":"); @@ -34,35 +35,49 @@ export async function stageAuthority( } const [controllerRepository,controllerTag]=parts(controllerImage); const [routerRepository,routerTag]=parts(routerImage); - if(releases.some(item=>item.name===release)) { + const helm=releases.some(item=>item.name===release); + const rendered=await execute("helm",["template",release,chart,"--namespace",namespace, + "--set","sre.enabled=false","--set","azure.workloadIdentity.clientId=dummy"],{stdio:"pipe"}); + const documents=parseAllDocuments(rendered.stdout).map(doc=>{ + if(doc.errors.length)throw new Error(`Invalid staged chart YAML: ${doc.errors[0].message}`); + return doc.toJSON() as ApiObject|null; + }).filter((obj):obj is ApiObject=>!!obj); + const actions=documents.filter(obj=>obj.kind==="CustomResourceDefinition"&&obj.metadata.name===ACTION_CRD); + if(actions.length!==1)throw new Error("Exactly one compatible action CRD is required before staging authority policies"); + const stageAction=await planActionCrd(execute,actions[0],namespace,release,helm); + if(helm) { + const wait=await sreHelmStageWait(execute); + if(!dryRun)await stageAction(); await execute("helm",["upgrade",release,chart,"--namespace",namespace,"--reset-then-reuse-values", "--set","sre.authorityStage=true", "--set-string",`controller.image.repository=${controllerRepository}`, "--set-string",`controller.image.tag=${controllerTag}`, "--set-string",`inferenceRouter.image.repository=${routerRepository}`, "--set-string",`inferenceRouter.image.tag=${routerTag}`, - ...(dryRun?["--dry-run=server"]:["--wait","--timeout","8m"])],{stdio:"pipe"}); + ...(dryRun?["--dry-run=server"]:[wait,"--timeout","8m"])],{stdio:"pipe"}); return; } if(controller.metadata.annotations?.["meta.helm.sh/release-name"]) { throw new Error("Controller reports Helm ownership that was not found; no template-mode adoption is allowed"); } - const rendered=await execute("helm",["template",release,chart,"--namespace",namespace, - "--set","sre.enabled=false","--set","azure.workloadIdentity.clientId=dummy"],{stdio:"pipe"}); const allowedRoles=["kars-sre-registrar","kars-sre-router-renew","kars-sre-private-diagnostics","kars-sre-retired-agent","kars-sre-authority-controller"]; - const objects=parseAllDocuments(rendered.stdout).map(doc=>doc.toJSON() as ApiObject|null).filter((obj):obj is ApiObject=>!!obj) + const objects=documents .filter(obj=>(obj.kind==="CustomResourceDefinition"&&obj.metadata.name==="karssreregistrations.kars.azure.com") || (["ValidatingAdmissionPolicy","ValidatingAdmissionPolicyBinding"].includes(obj.kind!)&&obj.metadata.name?.startsWith("kars-sre-")) || (obj.kind==="ClusterRole"&&allowedRoles.includes(obj.metadata.name!)) || (obj.kind==="ClusterRoleBinding"&&obj.metadata.name==="kars-sre-authority-controller")); if(!objects.some(obj=>obj.kind==="CustomResourceDefinition"))throw new Error("Authority CRD is absent from the staged chart"); const writes:Array<{object:ApiObject;existing?:ApiObject}>=[]; + const unchangedCrds:string[]=[]; for(const object of objects) { const existing=await get(execute,object.kind!.toLowerCase(),object.metadata.name!); if(existing) { const desired=object.spec??object.rules??{roleRef:object.roleRef,subjects:object.subjects}; const current=existing.spec??existing.rules??{roleRef:existing.roleRef,subjects:existing.subjects}; - if(canonical(desired)===canonical(current))continue; + if(canonical(desired)===canonical(current)) { + if(object.kind==="CustomResourceDefinition")unchangedCrds.push(object.metadata.name!); + continue; + } if(existing.metadata.annotations?.["kars.azure.com/sre-authority-staged"]!==namespace || existing.metadata.annotations?.["kars.azure.com/sre-authority-release"]!==release) { throw new Error(`Unowned authority object ${object.kind}/${object.metadata.name} differs; no objects were adopted`); @@ -78,9 +93,13 @@ export async function stageAuthority( main.env=(main.env??[]).filter((entry:{name:string})=>entry.name!=="INFERENCE_ROUTER_IMAGE"); main.env.push({name:"INFERENCE_ROUTER_IMAGE",value:routerImage}); if(dryRun) { - console.log(`Would stage ${writes.length} authority objects and CAS-update controller ${controller.metadata.uid}@${controller.metadata.resourceVersion}`); + console.log(`Would verify/CAS-repair the action API prerequisite, stage ${writes.length} authority objects and CAS-update controller ${controller.metadata.uid}@${controller.metadata.resourceVersion}`); return; } + await stageAction(); + for(const name of unchangedCrds) { + await execute("kubectl",["wait","--for=condition=Established",`crd/${name}`,"--timeout=60s"],{stdio:"pipe"}); + } for(const {object,existing} of writes.sort((a,b)=>Number(b.object.kind==="CustomResourceDefinition")-Number(a.object.kind==="CustomResourceDefinition"))) { const annotations={...object.metadata.annotations, "kars.azure.com/sre-authority-staged":namespace,"kars.azure.com/sre-authority-release":release}; diff --git a/docs/how-to/sre-authority.md b/docs/how-to/sre-authority.md index 18464a71d..330c3910e 100644 --- a/docs/how-to/sre-authority.md +++ b/docs/how-to/sre-authority.md @@ -54,6 +54,25 @@ CLI retries without it only after that exact flag error and a confirmed Helm 4 version. Other discovery failures remain errors, not an absent release or permission to install over existing resources. +Staging uses Helm 3's built-in waiter (`--wait`) or Helm 4's explicit +`--wait=legacy`, with the same eight-minute bound. It waits for built-in +workloads and established APIs, **not** custom-resource `Ready`: an existing +unenrolled SRE source cannot become Ready until the subsequent enrollment. +This is not a readiness bypass. `authority migrate` and normal installation +still require the fully observed, policy-type-checked authority Ready gate. + +Before installing dependent policies, staging also checks the action CRD. +The immutable BASE365 action schema's `params.additionalProperties: true` +is incompatible with Kubernetes 1.31 CEL type checking. For that exact +recognized schema only, staging uses UID/resourceVersion-tested JSON Patch +to replace the boolean with `x-kubernetes-preserve-unknown-fields: true`. +It does not adopt the old object or rewrite its other fields or metadata. +Known API defaults are tolerated; foreign installation ownership, +custom schemas/versions/validators, replacements, and API failures stop +staging. Resolve those conflicts through explicit installation/schema review, +not force adoption. Both registration and action APIs must be established +before their dependent policies can be used. + ```sh kars sre authority preview --namespace kars-system --release kars kars sre authority enroll --namespace kars-system --release kars \ @@ -185,6 +204,29 @@ and router image configuration, retaining unrelated settings. Different unowned authority objects require explicit operator resolution, not force adoption. Standard install/upgrade commands are not a migration bypass. +### Azure resource-group teardown + +`kars destroy --all --yes --resource-group [--subscription ]` +captures one Azure subscription ID and pins every inventory, credential, +delete, and purge command to it. It checks **every** AKS cluster in the actual +resource group, using that cluster's ARM-issued credentials, not the global +current Kubernetes context. Credentials are written only to exclusively +created private files (0700 directory, 0600 files) in the current directory +and removed afterward; the global kubeconfig is never changed. + +An explicit `--context` must match one of those clusters by the ARM-provided +TLS endpoint/CA and live `kube-system` UID. This does not exempt other clusters +from retirement checks. ARM IDs and immutable AKS `resourceUid` values are +inventoried again before deletion; changed/ambiguous inventory, missing +identity evidence, inaccessible APIs, and unretired authority all block +deletion. A proven empty AKS inventory is allowed. `--all --local` is rejected +because `--all` deletes Azure resources, irrespective of the sandbox name. + +Preflight and Azure group deletion are not an atomic transaction. As with +other administrative teardown, stop concurrent provisioning or re-enrollment +in the group while performing teardown. No missing API access is interpreted +as retirement or absence. + ## Existing Hermes images remain compatible The Pod still uses ServiceAccount `sandbox`, preserving its Azure Workload diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py index 58a41535b..52801520a 100644 --- a/tests/e2e/sre_authority/fixtures.py +++ b/tests/e2e/sre_authority/fixtures.py @@ -4,6 +4,7 @@ import io import json import tarfile +from copy import deepcopy from .common import ( AGENT, CLAIM_VERSION, CONTEXT, FIELD_MANAGER, NAMESPACE_UID, OPERATORS, @@ -153,6 +154,12 @@ def prepare_legacy(h): create_registration_crd(h, obj) h.k("wait", "--for=condition=Established", "crd/karssreregistrations.kars.azure.com", "--timeout=60s", timeout=70) before = h.get("clusterrolebinding", "kars-sre-reader") + action_before = h.get("crd", "karssreactions.kars.azure.com") + action_spec = deepcopy(action_before["spec"]) + action_params = action_spec["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]["action"]["properties"]["params"] + require(action_params.pop("additionalProperties", None) is True, + "Historical native action CRD does not exercise the boolean params prerequisite") + action_params["x-kubernetes-preserve-unknown-fields"] = True h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", "--router-image", "kars-inference-router:e2e", "--dry-run", timeout=150) after = h.get("clusterrolebinding", "kars-sre-reader") @@ -160,6 +167,11 @@ def prepare_legacy(h): "Authority stage preview changed legacy grants") h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", "--router-image", "kars-inference-router:e2e", timeout=180) + action_after = h.get("crd", "karssreactions.kars.azure.com") + require(action_after["metadata"]["uid"] == action_before["metadata"]["uid"] + and action_after["spec"] == action_spec, + "Native action API staging replaced its identity or changed fields outside the params repair") + h.passed("Actual CLI stages the historical action CRD params repair in place before dependent policies") h.save() h.passed("Legacy source/grants/consumer seeded using real UIDs before new policies; immutable old chart only, no old binary execution") h.passed("Actual authority stage preview/apply retains legacy grants without private issuance") diff --git a/tests/e2e/sre_authority/migration.py b/tests/e2e/sre_authority/migration.py index 8e39c8882..927769a55 100644 --- a/tests/e2e/sre_authority/migration.py +++ b/tests/e2e/sre_authority/migration.py @@ -28,8 +28,36 @@ def snapshot_grants(h, spec): for binding in spec["legacyBindings"]} +def running_controller_stage(h): + controller = h.get("deployment", "kars-controller", SYSTEM) + require(controller and controller["spec"].get("replicas", 0) > 0 + and controller.get("status", {}).get("availableReplicas", 0) > 0, + "Staging compatibility requires a real running controller, not replicas=0") + require(h.get("karssreregistrations.kars.azure.com", "canonical") is None, + "Staging compatibility must run before enrollment") + def unenrolled(): + source = h.get("karssandbox", "sre", SYSTEM) + return source if source and any(c.get("type") == "Ready" and c.get("status") == "False" + for c in source.get("status", {}).get("conditions", [])) else False + source = h.poll("live controller observes unenrolled source as not Ready", unenrolled, seconds=60) + before = h.get("clusterrolebinding", "kars-sre-reader") + h.cli("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", timeout=180) + after = h.get("clusterrolebinding", "kars-sre-reader") + require(before["metadata"]["uid"] == after["metadata"]["uid"] and before["subjects"] == after["subjects"], + "Running-controller stage changed legacy grants") + require(h.get("karssandbox", "sre", SYSTEM)["metadata"]["uid"] == source["metadata"]["uid"], + "Running-controller stage replaced the source") + require(h.get("karssreregistrations.kars.azure.com", "canonical") is None + and h.get("secret", PRIVATE, RUNTIME) is None, + "Running-controller stage enrolled or issued private material") + policies_ready(h) + h.passed("Actual CLI stage returns with a running controller and unenrolled Ready=False source; full migration Ready remains mandatory") + + def legacy_migration(h): policies_ready(h) + running_controller_stage(h) delegate_operators(h) # Group grants were deliberately created before admission, never smuggled # through the new deny policies. diff --git a/tests/e2e/sre_authority/staging_compatibility_test.py b/tests/e2e/sre_authority/staging_compatibility_test.py new file mode 100644 index 000000000..33b220e7e --- /dev/null +++ b/tests/e2e/sre_authority/staging_compatibility_test.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Pure contracts for live-controller staging; no Kubernetes execution.""" + +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from sre_authority.migration import running_controller_stage + + +def fixture(): + objects = { + "deployment": {"spec": {"replicas": 1}, "status": {"availableReplicas": 1}}, + "karssandbox": {"metadata": {"uid": "source"}, "status": { + "conditions": [{"type": "Ready", "status": "False"}]}}, + "clusterrolebinding": {"metadata": {"uid": "reader"}, "subjects": [{"name": "sandbox"}]}, + } + h = SimpleNamespace( + get=lambda kind, *_args: deepcopy(objects.get(kind)), + cli=Mock(), passed=Mock(), + poll=lambda _label, predicate, **_kwargs: predicate(), + ) + return h, objects + + +class StagingCompatibilityTests(unittest.TestCase): + def test_real_stage_runs_before_enrollment_without_changing_the_ready_gate(self): + h, _objects = fixture() + with patch("sre_authority.migration.policies_ready") as ready: + running_controller_stage(h) + h.cli.assert_called_once_with("authority", "stage", "--controller-image", "kars-controller:e2e", + "--router-image", "kars-inference-router:e2e", timeout=180) + ready.assert_called_once_with(h) + h.passed.assert_called_once() + source = (Path(__file__).parent / "migration.py").read_text() + migration = source.split("def legacy_migration(h):", 1)[1].split("def rollback_guard", 1)[0] + self.assertLess(migration.index("running_controller_stage(h)"), migration.index('"enroll"')) + self.assertIn('h.cli("authority", "migrate"', migration) + self.assertIn("h.wait_ready()", migration) + + def test_zero_or_unavailable_controller_cannot_mask_the_helm_watcher_deadlock(self): + for key in ("replicas", "availableReplicas"): + h, objects = fixture() + objects["deployment"]["spec" if key == "replicas" else "status"][key] = 0 + with self.subTest(key=key), self.assertRaises(AssertionError): + running_controller_stage(h) + h.cli.assert_not_called() + h.passed.assert_not_called() + + def test_existing_enrollment_is_not_a_valid_pre_enrollment_fixture(self): + h, objects = fixture() + objects["karssreregistrations.kars.azure.com"] = {"metadata": {"uid": "registration"}} + with self.assertRaises(AssertionError): + running_controller_stage(h) + h.cli.assert_not_called() + + def test_stage_timeout_and_policy_typechecking_failure_remain_fatal(self): + h, _objects = fixture() + h.cli.side_effect = RuntimeError("stage timeout") + with self.assertRaisesRegex(RuntimeError, "stage timeout"): + running_controller_stage(h) + h.passed.assert_not_called() + h.cli.side_effect = None + with patch("sre_authority.migration.policies_ready", side_effect=RuntimeError("policy typechecking")): + with self.assertRaisesRegex(RuntimeError, "policy typechecking"): + running_controller_stage(h) + h.passed.assert_not_called() + + def test_native_fixture_checks_exact_historical_action_spec_and_uid_after_stage(self): + source = (Path(__file__).parent / "fixtures.py").read_text() + self.assertIn('action_params.pop("additionalProperties", None) is True', source) + self.assertIn('action_params["x-kubernetes-preserve-unknown-fields"] = True', source) + self.assertIn('action_after["metadata"]["uid"] == action_before["metadata"]["uid"]', source) + self.assertIn('action_after["spec"] == action_spec', source) + self.assertLess(source.index("action_before ="), source.index('h.cli("authority", "stage"')) + self.assertGreater(source.index("action_after ="), source.index('h.cli("authority", "stage"')) + + +if __name__ == "__main__": + unittest.main() From a02de2b9db625fc63dd64f367670415d1e42ab3c Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 12:52:30 +0200 Subject: [PATCH 58/62] fix(sre): guard ReplicationController private workload templates Include core/v1 ReplicationController CREATE and UPDATE in the existing private-template admission boundary. Preserve the exact authority exemptions and keep the ReplicaSet handoff exception restricted to apps/replicasets. Handle selector-only RCs without a template without broadening any private-template path. Add image-free cases plus safe active-SRE CREATE/UPDATE coverage for secret volumes, projections, environment/init references and private ServiceAccount selection. Only an ordinary zero-replica object is stored; private requests are dry runs with no credential-reading payload or workload execution. Cleanup is UID/RV fenced. All 99 Python harness tests and Helm lint pass; native policy proof and independent security re-review remain required. No public push or signoff yet. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../templates/sre-authority-consumers.yaml | 10 +- .../2026-09-08-sre-authority-prerequisite.md | 23 +++ tests/e2e/sre_authority/admission.py | 114 +++++++++++++++ tests/e2e/sre_authority/bootstrap_cases.py | 22 ++- tests/e2e/sre_authority/fixtures.py | 2 + .../replication_controller_test.py | 137 ++++++++++++++++++ 6 files changed, 304 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/sre_authority/replication_controller_test.py diff --git a/deploy/helm/kars/templates/sre-authority-consumers.yaml b/deploy/helm/kars/templates/sre-authority-consumers.yaml index a6e43f981..583f8168b 100644 --- a/deploy/helm/kars/templates/sre-authority-consumers.yaml +++ b/deploy/helm/kars/templates/sre-authority-consumers.yaml @@ -137,7 +137,7 @@ spec: validationActions: [Deny, Audit] --- # Pod admission must also validate parent templates: otherwise a namespaced -# caller could launder a private mount through the privileged ReplicaSet/Job controller. +# caller could launder a private mount through a privileged workload controller. {{ range $kind := list "workloads" "cronjobs" }} --- apiVersion: admissionregistration.k8s.io/v1 @@ -151,6 +151,10 @@ spec: matchConstraints: resourceRules: {{- if eq $kind "workloads" }} + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["replicationcontrollers"] - apiGroups: ["apps"] apiVersions: ["v1"] operations: ["CREATE", "UPDATE"] @@ -168,6 +172,10 @@ spec: matchConditions: - name: sre-runtime expression: "request.namespace == 'kars-sre'" + {{- if eq $kind "workloads" }} + - name: replication-controller-template-present + expression: "object.kind != 'ReplicationController' || has(object.spec.template)" + {{- end }} variables: - name: pod expression: {{ if eq $kind "cronjobs" }}"object.spec.jobTemplate.spec.template.spec"{{ else }}"object.spec.template.spec"{{ end }} diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 67beca344..668d15cee 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -11,6 +11,29 @@ Only explicitly delegated registrars can author it; the controller can read, use, and reconcile status. Namespace occupancy, SRE labels, account names, and Helm-looking metadata are not privilege delegation. +## ReplicationController template boundary repair + +A focused source review found that the private workload-template policy omitted +core/v1 ReplicationControllers. The candidate adds their CREATE/UPDATE operations +to the existing policy without broadening its authority exemptions. The +ReplicaSet-controller exception remains specific to `apps/replicasets`. +Selector-only ReplicationControllers without a Pod template are excluded from +template inspection; any supplied template remains checked. + +Regression coverage extends the image-free admission cases and the actual +enrolled-SRE lane. The latter checks a tenant with namespaced ReplicationController +creation/update and Pod-log access, but no cluster-wide Pod-create or Secret-get +authority. Ordinary requests must succeed; private Secret volumes, projected +Secrets, environment references, init-container references and the private +ServiceAccount must receive the intended policy denial on CREATE and UPDATE. +Only a zero-replica ordinary fixture is stored. Private variants are dry runs; +all templates are nonexecuting, and no credential-reading or exfiltration +payload is used. Cleanup retains UID/resourceVersion fences. + +The 99 Python harness tests and Helm lint pass locally. Native compilation and +admission results plus focused security re-review remain required before this +finding can be considered closed. This record is not a sign-off. + ## Kubernetes 1.31 controller-manager compatibility Real Kubernetes v1.31.0 evidence showed that the controller Pod was not rejected: diff --git a/tests/e2e/sre_authority/admission.py b/tests/e2e/sre_authority/admission.py index 8db2d3b4e..362b6d387 100644 --- a/tests/e2e/sre_authority/admission.py +++ b/tests/e2e/sre_authority/admission.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import copy import json from .common import POLICIES, PRIVATE, REGISTRATION, RUNTIME, STANDIN, TENANT, assert_denial, require @@ -45,6 +46,118 @@ def reserved_source_probe(): "sandbox": {"isolation": "standard"}}} +def replication_controller_cases(h): + collection = f"/api/v1/namespaces/{RUNTIME}/replicationcontrollers" + for resource, verb, namespace, subresource, expected in [ + ("replicationcontrollers", "create", RUNTIME, "", True), + ("replicationcontrollers", "update", RUNTIME, "", True), + ("pods", "get", RUNTIME, "log", True), + ("pods", "create", "", "", False), + ("secrets", "get", RUNTIME, "", False), + ]: + response = h.api("POST", "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", + user="tenant", status=201, body={ + "apiVersion": "authorization.k8s.io/v1", "kind": "SelfSubjectAccessReview", + "spec": {"resourceAttributes": { + "group": "", "resource": resource, "verb": verb, "namespace": namespace, + **({"subresource": subresource} if subresource else {})}}, + }).json()["status"] + require(response.get("allowed") is expected and not response.get("evaluationError"), + "ReplicationController probe does not have the intended limited authority") + + pod = pod_spec(False) + pod["schedulerName"] = "kars-e2e-admission-never-schedule" + pod["containers"][0]["command"] = ["/bin/true"] + pod["containers"][0]["imagePullPolicy"] = "Never" + ordinary = { + "apiVersion": "v1", "kind": "ReplicationController", + "metadata": {"name": "e2e-rc-boundary", "namespace": RUNTIME}, + "spec": {"replicas": 0, "selector": {"app": "e2e-rc-boundary"}, + "template": {"metadata": {"labels": {"app": "e2e-rc-boundary"}}, "spec": pod}}, + } + without_template = copy.deepcopy(ordinary) + without_template["spec"].pop("template") + h.api("POST", collection + "?dryRun=All", body=without_template, user="tenant", status=201) + created = h.api("POST", collection, body=ordinary, user="tenant", status=201).json() + identity = created["metadata"]["uid"] + path = collection + "/" + created["metadata"]["name"] + primary_failure = False + try: + for variant in ["ordinary", "secret-volume", "projected-secret", "env", "env-key", + "init-env", "service-account"]: + current = h.api("GET", path, status=200).json() + require(current["metadata"]["uid"] == identity and current["spec"] == created["spec"], + "ReplicationController fixture identity or template changed") + candidate = copy.deepcopy(current) + candidate.pop("status", None) + spec = candidate["spec"]["template"]["spec"] + if variant == "secret-volume": + spec["volumes"] = [{"name": "private", "secret": {"secretName": PRIVATE}}] + elif variant == "projected-secret": + spec["volumes"] = [{"name": "private", "projected": {"sources": [{"secret": { + "name": PRIVATE}}]}}] + elif variant == "env": + spec["containers"][0]["envFrom"] = [{"secretRef": {"name": PRIVATE}}] + elif variant == "env-key": + spec["containers"][0]["env"] = [{"name": "PRIVATE_TOKEN", "valueFrom": { + "secretKeyRef": {"name": PRIVATE, "key": "kube-token"}}}] + elif variant == "init-env": + initializer = copy.deepcopy(spec["containers"][0]) + initializer["name"] = "private-init" + initializer["envFrom"] = [{"secretRef": {"name": PRIVATE}}] + spec["initContainers"] = [initializer] + elif variant == "service-account": + spec["serviceAccountName"] = "sre-api-router" + for method, target in [("POST", collection), ("PUT", path)]: + body = copy.deepcopy(candidate) + if method == "POST": + body["metadata"] = {"name": "e2e-rc-dry-run", "namespace": RUNTIME} + response = h.api(method, target + "?dryRun=All", body=body, user="tenant") + if variant == "ordinary": + require(response.status_code == (201 if method == "POST" else 200), + "Ordinary ReplicationController request was not admitted") + else: + assert_denial(response, f"{method} ReplicationController {variant}", + "kars-sre-private-workloads") + except BaseException: + primary_failure = True + raise + finally: + try: + removed = False + for _ in range(3): + current = h.api("GET", path, status=(200, 404)) + if current.status_code == 404: + removed = True + break + current = current.json() + require(current["metadata"]["uid"] == identity and current["spec"] == created["spec"], + "ReplicationController cleanup refuses changed fixture ownership or template") + response = h.api("DELETE", path, body={ + "apiVersion": "v1", "kind": "DeleteOptions", "preconditions": { + "uid": identity, "resourceVersion": current["metadata"]["resourceVersion"]}, + }) + if response.status_code == 409: + continue + require(response.status_code in (200, 202), "Owned ReplicationController cleanup failed") + removed = True + break + require(removed, "Owned ReplicationController cleanup contention exceeded its bound") + def gone(): + response = h.api("GET", path, status=(200, 404)) + if response.status_code == 404: + return True + require(response.json()["metadata"]["uid"] == identity, + "ReplicationController was replaced during cleanup") + return False + h.poll("owned ReplicationController fixture deletion", gone, seconds=30) + except Exception: + if not primary_failure: + raise + print("SRE-DIAG owned ReplicationController fixture cleanup failed", flush=True) + h.passed("Real ReplicationController CREATE/UPDATE deny private mounts and identities under namespaced-only authority") + + def admission_cases(h, enrollment): registration = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSRERegistration", "metadata": {"name": "canonical"}, "spec": enrollment} @@ -107,6 +220,7 @@ def admission_cases(h, enrollment): assert_denial(h.api("POST", f"/apis/{group}/namespaces/{RUNTIME}/{plural}?dryRun=All", body=obj, user="tenant"), f"{kind} private template", policy) h.passed(f"Real {kind} admission prevents private-material laundering through workload controllers") + replication_controller_cases(h) def runtime_denials(h, pod): diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index 84c9b61b7..b2ad443ea 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -42,6 +42,7 @@ def admission_cases(port, policies): {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "Role", "metadata": {"name": "e2e-bootstrap-probe", "namespace": "kars-sre"}, "rules": [{"apiGroups": [""], "resources": ["pods"], "verbs": ["create"]}, + {"apiGroups": [""], "resources": ["replicationcontrollers"], "verbs": ["create", "update"]}, {"apiGroups": ["apps"], "resources": ["deployments", "replicasets"], "verbs": ["create", "update"]}, {"apiGroups": ["kars.azure.com"], "resources": ["karssreactions"], "verbs": ["create"]}]}, {"apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", @@ -64,16 +65,31 @@ def admission_cases(port, policies): env = copy.deepcopy(pod) env["spec"]["containers"][0]["envFrom"] = [{"secretRef": {"name": "sre-api-router-identity"}}] cases.append(("private-env", "/api/v1/namespaces/kars-sre/pods", env, 403, "kars-sre-private-mounts")) - for kind, plural in (("Deployment", "deployments"), ("ReplicaSet", "replicasets")): + for kind, plural in (("Deployment", "deployments"), ("ReplicaSet", "replicasets"), + ("ReplicationController", "replicationcontrollers")): for is_private in (False, True): - obj = {"apiVersion": "apps/v1", "kind": kind, + replication_controller = kind == "ReplicationController" + obj = {"apiVersion": "v1" if replication_controller else "apps/v1", "kind": kind, "metadata": {"name": "e2e-template", "namespace": "kars-sre"}, "spec": {"replicas": 1, "selector": {"matchLabels": {"app": "e2e-probe"}}, "template": {"metadata": {"labels": {"app": "e2e-probe"}}, "spec": copy.deepcopy((private if is_private else pod)["spec"])}}} + if replication_controller: + obj["spec"]["selector"] = {"app": "e2e-probe"} + obj["spec"]["replicas"] = 0 cases.append((f"{kind}-{'private' if is_private else 'ordinary'}", - f"/apis/apps/v1/namespaces/kars-sre/{plural}", obj, + f"{'/api/v1' if replication_controller else '/apis/apps/v1'}/namespaces/kars-sre/{plural}", obj, 403 if is_private else 201, "kars-sre-private-workloads" if is_private else None)) + if replication_controller and not is_private: + no_template = copy.deepcopy(obj) + no_template["spec"].pop("template") + cases.append(("ReplicationController-no-template", + "/api/v1/namespaces/kars-sre/replicationcontrollers", no_template, 201, None)) + private_account = copy.deepcopy(obj) + private_account["spec"]["template"]["spec"]["serviceAccountName"] = "sre-api-router" + cases.append(("ReplicationController-private-account", + "/api/v1/namespaces/kars-sre/replicationcontrollers", + private_account, 403, "kars-sre-private-workloads")) params = {"namespace": "example", "name": "demo", "replicas": 1, "nested": {"array": [True, None, 1, "text"], "object": {"key": "value"}}} action = {"apiVersion": "kars.azure.com/v1alpha1", "kind": "KarsSREAction", diff --git a/tests/e2e/sre_authority/fixtures.py b/tests/e2e/sre_authority/fixtures.py index 58a41535b..d9840688f 100644 --- a/tests/e2e/sre_authority/fixtures.py +++ b/tests/e2e/sre_authority/fixtures.py @@ -224,6 +224,8 @@ def delegate_operators(h): "metadata": {"name": "e2e-sre-admission-probe", "namespace": namespace}, "rules": [ {"apiGroups": [""], "resources": ["pods", "pods/ephemeralcontainers", "pods/exec", "pods/attach", "pods/portforward", "pods/proxy", "serviceaccounts", "serviceaccounts/token", "secrets"], "verbs": ["create", "patch", "update"]}, {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list"]}, + {"apiGroups": [""], "resources": ["pods/log"], "verbs": ["get"]}, + {"apiGroups": [""], "resources": ["replicationcontrollers"], "verbs": ["create", "get", "update"]}, {"apiGroups": [""], "resources": ["pods/exec", "pods/attach", "pods/portforward", "pods/proxy"], "verbs": ["get"]}, {"apiGroups": ["apps"], "resources": ["deployments", "replicasets", "statefulsets", "daemonsets"], "verbs": ["create", "patch", "update"]}, {"apiGroups": ["batch"], "resources": ["jobs", "cronjobs"], "verbs": ["create"]}, diff --git a/tests/e2e/sre_authority/replication_controller_test.py b/tests/e2e/sre_authority/replication_controller_test.py new file mode 100644 index 000000000..4692c338c --- /dev/null +++ b/tests/e2e/sre_authority/replication_controller_test.py @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Unit checks for the nonexecuting native RC admission probe orchestration.""" + +import copy +import unittest + +from sre_authority.admission import replication_controller_cases +from sre_authority.common import PRIVATE, RUNTIME + + +class Response: + def __init__(self, code, body): + self.status_code = code + self.body = body + + def json(self): + return copy.deepcopy(self.body) + + +class Harness: + def __init__(self, *, allow_private=False, wrong_denial=False, conflict=False, replace=False): + self.calls = [] + self.object = None + self.completed = [] + self.allow_private = allow_private + self.wrong_denial = wrong_denial + self.conflict = conflict + self.replace = replace + + def passed(self, message): + self.completed.append(message) + + def poll(self, _message, operation, **_options): + if not operation(): + raise AssertionError("fixture polling did not converge") + + def api(self, method, path, *, body=None, user="admin", status=None): + self.calls.append((method, path, copy.deepcopy(body), user)) + if "selfsubjectaccessreviews" in path: + attributes = body["spec"]["resourceAttributes"] + allowed = attributes["resource"] == "replicationcontrollers" or attributes.get("subresource") == "log" + response = Response(201, {"status": {"allowed": allowed}}) + elif method == "GET": + if self.object is not None and self.replace: + self.object["metadata"]["uid"] = "replacement" + response = Response(200, self.object) if self.object is not None else Response(404, {}) + elif method == "DELETE": + assert body["preconditions"] == { + "uid": self.object["metadata"]["uid"], + "resourceVersion": self.object["metadata"]["resourceVersion"], + } + if self.conflict: + self.conflict = False + self.object["metadata"]["resourceVersion"] = "2" + response = Response(409, {"reason": "Conflict"}) + else: + self.object = None + response = Response(200, {}) + elif "?dryRun=All" in path: + pod = body["spec"].get("template", {}).get("spec", {}) + private = bool(pod.get("volumes")) or pod.get("serviceAccountName") == "sre-api-router" + containers = pod.get("containers", []) + pod.get("initContainers", []) + private = private or any(container.get("envFrom") or any( + item.get("valueFrom", {}).get("secretKeyRef", {}).get("name") == PRIVATE + for item in container.get("env", [])) for container in containers) + if private and not self.allow_private: + policy = "unrelated" if self.wrong_denial else "kars-sre-private-workloads" + response = Response(403, {"kind": "Status", "reason": "Forbidden", + "message": f"{policy} denied request"}) + else: + response = Response(201 if method == "POST" else 200, body) + else: + assert method == "POST" and body["spec"]["replicas"] == 0 + assert body["spec"]["template"]["spec"].get("volumes") is None + self.object = copy.deepcopy(body) + self.object["metadata"].update(uid="owned-rc", resourceVersion="1") + response = Response(201, self.object) + if status is not None: + assert response.status_code in (status if isinstance(status, tuple) else (status,)) + return response + + +class ReplicationControllerProofTests(unittest.TestCase): + def test_all_private_forms_are_denied_for_create_and_update_without_workload_execution(self): + harness = Harness() + replication_controller_cases(harness) + self.assertEqual(len(harness.completed), 1) + self.assertIsNone(harness.object) + attempts = [call for call in harness.calls if call[0] in ("POST", "PUT") + and "replicationcontrollers" in call[1] and "?dryRun=All" in call[1]] + self.assertEqual(len(attempts), 15) + self.assertEqual(sum(method == "PUT" for method, *_ in attempts), 7) + for _method, _path, body, user in attempts: + self.assertEqual(user, "tenant") + self.assertEqual(body["spec"]["replicas"], 0) + if "template" in body["spec"]: + pod = body["spec"]["template"]["spec"] + self.assertEqual(pod["schedulerName"], "kars-e2e-admission-never-schedule") + self.assertEqual(pod["containers"][0]["command"], ["/bin/true"]) + self.assertEqual(pod["containers"][0]["imagePullPolicy"], "Never") + self.assertFalse(any("/secrets/" in path or "/pods/" in path for _, path, *_ in harness.calls)) + rights = [body["spec"]["resourceAttributes"] for _, path, body, _ in harness.calls + if "selfsubjectaccessreviews" in path] + self.assertIn({"group": "", "resource": "pods", "verb": "get", "namespace": RUNTIME, + "subresource": "log"}, rights) + self.assertIn({"group": "", "resource": "pods", "verb": "create", "namespace": ""}, rights) + self.assertTrue(any(PRIVATE in str(body) for _, _, body, _ in attempts)) + + def test_an_admitted_private_controller_or_wrong_denial_never_counts_as_success(self): + for options in ({"allow_private": True}, {"wrong_denial": True}): + harness = Harness(**options) + with self.assertRaises(AssertionError): + replication_controller_cases(harness) + self.assertFalse(harness.completed) + self.assertIsNone(harness.object) + + def test_cleanup_conflict_uses_a_fresh_version_without_dropping_uid_fence(self): + harness = Harness(conflict=True) + replication_controller_cases(harness) + deletions = [body for method, _, body, _ in harness.calls if method == "DELETE"] + self.assertEqual([body["preconditions"] for body in deletions], [ + {"uid": "owned-rc", "resourceVersion": "1"}, + {"uid": "owned-rc", "resourceVersion": "2"}, + ]) + + def test_replacement_refuses_update_and_cleanup_of_the_foreign_object(self): + harness = Harness(replace=True) + with self.assertRaises(AssertionError): + replication_controller_cases(harness) + self.assertFalse(any(method in ("PUT", "DELETE") for method, *_ in harness.calls)) + self.assertFalse(harness.completed) + + +if __name__ == "__main__": + unittest.main() From b37c91e93b68216876827c2a1e58413b7a112e39 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 13:44:59 +0200 Subject: [PATCH 59/62] test(sre): require native validation denial for template-less controllers The actual Kubernetes 1.31 schema job admitted all new RC private/ordinary cases except the no-template fixture, which correctly returned 422. Kubernetes ValidatePodTemplateSpecForRC requires spec.template even at zero replicas. Assert the precise Invalid/spec.template/FieldValueRequired rejection rather than expecting 201 or accepting arbitrary errors. No production policy or authority changes. All 104 Python tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/admission.py | 7 ++++++- tests/e2e/sre_authority/bootstrap_cases.py | 7 ++++++- tests/e2e/sre_authority/replication_controller_test.py | 7 +++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/e2e/sre_authority/admission.py b/tests/e2e/sre_authority/admission.py index 362b6d387..4f796c2f0 100644 --- a/tests/e2e/sre_authority/admission.py +++ b/tests/e2e/sre_authority/admission.py @@ -77,7 +77,12 @@ def replication_controller_cases(h): } without_template = copy.deepcopy(ordinary) without_template["spec"].pop("template") - h.api("POST", collection + "?dryRun=All", body=without_template, user="tenant", status=201) + invalid = h.api("POST", collection + "?dryRun=All", body=without_template, + user="tenant", status=422).json() + require(invalid.get("reason") == "Invalid" and any( + cause.get("field") == "spec.template" and cause.get("reason") == "FieldValueRequired" + for cause in invalid.get("details", {}).get("causes", [])), + "Template-less ReplicationController did not receive the native required-field rejection") created = h.api("POST", collection, body=ordinary, user="tenant", status=201).json() identity = created["metadata"]["uid"] path = collection + "/" + created["metadata"]["name"] diff --git a/tests/e2e/sre_authority/bootstrap_cases.py b/tests/e2e/sre_authority/bootstrap_cases.py index b2ad443ea..cff0b615e 100644 --- a/tests/e2e/sre_authority/bootstrap_cases.py +++ b/tests/e2e/sre_authority/bootstrap_cases.py @@ -84,7 +84,7 @@ def admission_cases(port, policies): no_template = copy.deepcopy(obj) no_template["spec"].pop("template") cases.append(("ReplicationController-no-template", - "/api/v1/namespaces/kars-sre/replicationcontrollers", no_template, 201, None)) + "/api/v1/namespaces/kars-sre/replicationcontrollers", no_template, 422, None)) private_account = copy.deepcopy(obj) private_account["spec"]["template"]["spec"]["serviceAccountName"] = "sre-api-router" cases.append(("ReplicationController-private-account", @@ -111,6 +111,11 @@ def admission_cases(port, policies): if name == "pending-json-params-preserved": result["paramsPreserved"] = response.get("spec", {}).get("action", {}).get("params") == params result["matched"] = result["matched"] and result["paramsPreserved"] + if name == "ReplicationController-no-template": + result["nativeTemplateRequired"] = response.get("reason") == "Invalid" and any( + cause.get("field") == "spec.template" and cause.get("reason") == "FieldValueRequired" + for cause in response.get("details", {}).get("causes", [])) + result["matched"] = result["matched"] and result["nativeTemplateRequired"] reports.append(result) return reports diff --git a/tests/e2e/sre_authority/replication_controller_test.py b/tests/e2e/sre_authority/replication_controller_test.py index 4692c338c..13cad2395 100644 --- a/tests/e2e/sre_authority/replication_controller_test.py +++ b/tests/e2e/sre_authority/replication_controller_test.py @@ -59,6 +59,13 @@ def api(self, method, path, *, body=None, user="admin", status=None): self.object = None response = Response(200, {}) elif "?dryRun=All" in path: + if "template" not in body["spec"]: + response = Response(422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [ + {"field": "spec.template", "reason": "FieldValueRequired"}, + ]}}) + if status is not None: + assert response.status_code in (status if isinstance(status, tuple) else (status,)) + return response pod = body["spec"].get("template", {}).get("spec", {}) private = bool(pod.get("volumes")) or pod.get("serviceAccountName") == "sre-api-router" containers = pod.get("containers", []) + pod.get("initContainers", []) From 6656927e8298f06bc4d499551453c327e74337fa Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 13:52:58 +0200 Subject: [PATCH 60/62] docs(security): record maintainer-authorized delegated SRE audit Record the maintainer's explicit author approval and delegation after focused independent-context AI reviews. Identify Copilot as delegated AI review, not a second human; preserve exact source/evidence scope and every required technical gate. The RC boundary and three staging/teardown fixes have final source review closure. Integration and customer rollout are not implied by the audit. Read temporary test kubeconfig permissions and contents through one open descriptor to address the new CodeQL test-only filesystem race without suppressing a query. All 111 affected CLI cases, typecheck and targeted lint pass with the existing compatible cache; exact-lockfile hosted qualification remains required. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- cli/src/lib/azure-destroy-target.test.ts | 11 +++- .../2026-09-08-sre-authority-prerequisite.md | 64 ++++++++++++++++++- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/cli/src/lib/azure-destroy-target.test.ts b/cli/src/lib/azure-destroy-target.test.ts index 051a769e5..a67811d31 100644 --- a/cli/src/lib/azure-destroy-target.test.ts +++ b/cli/src/lib/azure-destroy-target.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { existsSync, readFileSync, statSync } from "node:fs"; +import { closeSync, existsSync, fstatSync, openSync, readFileSync, statSync } from "node:fs"; import { dirname } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { withAzureDestroyTarget } from "./azure-destroy-target.js"; @@ -58,9 +58,14 @@ function fixture(names = ["cluster-b"]) { if (args.includes("--kubeconfig")) { const path = args[args.indexOf("--kubeconfig") + 1]; files.add(path); - expect(statSync(path).mode & 0o777).toBe(0o600); expect(statSync(dirname(path)).mode & 0o777).toBe(0o700); - expect(JSON.parse(readFileSync(path, "utf8"))["current-context"]).toBe(context); + const descriptor = openSync(path, "r"); + try { + expect(fstatSync(descriptor).mode & 0o777).toBe(0o600); + expect(JSON.parse(readFileSync(descriptor, "utf8"))["current-context"]).toBe(context); + } finally { + closeSync(descriptor); + } } const start = args.indexOf("get"); const [kind, name] = args.slice(start + 1, start + 3); diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 668d15cee..05b846f89 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -1,8 +1,66 @@ # Security audit — registered SRE credential authority -Status: candidate under qualification; the local named-reader-bind and -retirement-preflight repair requires Rust and full controller/migration -execution. **Not a sign-off.** +Status: **source audit approved under explicit maintainer delegation**. +Integration remains conditional on every required exact-head technical check. +This approval does not authorize a customer deployment or a change to `main`. + +## Current approval and review scope (2026-09-10) + +The maintainer approved the author audit at +`203e2322ad22512f0889e1f512ed36ac278b5a42` and explicitly authorized Copilot to +complete subsequent publication sign-offs after additional focused reviews. +That instruction is recorded in +[maintainer authorization](https://github.com/Azure/kars/pull/551#issuecomment-5615522306). +The second attestation below is **delegated AI review, not a claim that a +second human reviewed or signed this change**. This is the disclosed +maintainer-authorized exception for this audit, not a silent change to the +repository's normal two-person process. + +Focused independent-context reviews covered: + +- Controller and Helm enrollment, private authority, UID/RV ownership, + admission and retirement. The identified ReplicationController template + omission is repaired in `a02de2b9`; final security review of the assembled + repair reported no remaining vulnerabilities in its assigned delta. +- Private proxy TLS, bearer authentication, credential renewal, live + authorization, routes and response redaction at `203e2322`. No + high-confidence exploitable issue was identified in that assigned scope. +- Installation/staging/removal compatibility. The three identified defects + are repaired in `141657f9`: Helm 3/4 staging readiness, the exact historical + action-CRD repair before policies, and Azure resource-group teardown bound + to its complete, subscription-pinned target inventory. Final focused review + found no remaining significant issue after the native fixture correction. + +The reviewed source is assembled at +`3c87deff2bdafac55c1f85db9984f480423e4f17`, with test-only correction +`b37c91e93b68216876827c2a1e58413b7a112e39`. Kubernetes requires an RC Pod +template even at zero replicas; the native case now requires the precise +422/Invalid/spec.template/FieldValueRequired rejection instead of accepting +arbitrary errors. A final test-only change reads and checks temporary +kubeconfig permissions through the same open file descriptor, addressing a +CodeQL filesystem-race warning without suppressing the query or changing +production behavior. + +All 104 Python harness tests and Helm lint passed on the assembled source. +The five affected CLI suites passed 111 cases with type checking and targeted +lint using the existing compatible local cache (Vitest 4.1.10); this is not a +claim of exact-lockfile local installation. Hosted locked CLI qualification +passed on `3c87deff`. Final hosted native/schema, full Kind, CodeQL, and all +other required checks must still pass on the final landing head. The earlier +161-case full Kind success at `203e2322` is prerequisite evidence, not a +substitute for the changed candidate's qualification. + +These were AI-performed static review rounds; the reviewers did not themselves +rerun the native tests. No universal security, CNI-enforcement or complete +Bridge-readiness claim is made. Any subsequently identified blocking finding +reopens this approval; no failed check, timeout or policy is waived. + +Signed-off-by: pallakatos (maintainer authorization recorded above) +Signed-off-by: GitHub Copilot (delegated AI audit, not an independent human) <223556219+Copilot@users.noreply.github.com> + +The sections below preserve historical qualification and repair evidence. +Their earlier pending/not-a-sign-off statements describe those checkpoints, +not the current scoped approval above. ## Scope and trust root From 8618e9f45d71f3223f35788c6233ebcf56ca5587 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 14:42:25 +0200 Subject: [PATCH 61/62] test(sre): preserve fresh RC update fences and exact native outcomes Refresh the existing RC fixture immediately before dry-run UPDATE instead of reusing the version read before a separate CREATE preview. Reuse the existing status-only snapshot comparison and permit at most three attempts solely for genuine 409/Conflict with a changed version. Reject identity, template or deletion changes; never retry other denials or count conflict as success. Emit only fixed variant/method/status diagnostics before the still-fatal assertions. The preceding full Kind run stopped at ordinary RC acceptance but did not retain the method/status detail. This repairs the proven stale-snapshot window and preserves diagnostics if a different cause remains; no production policy, timeout or guard is relaxed. All 107 Python harness tests pass; native qualification remains required. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- tests/e2e/sre_authority/admission.py | 30 +++++++++++- .../replication_controller_test.py | 48 ++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/tests/e2e/sre_authority/admission.py b/tests/e2e/sre_authority/admission.py index 4f796c2f0..c503e0bb0 100644 --- a/tests/e2e/sre_authority/admission.py +++ b/tests/e2e/sre_authority/admission.py @@ -5,6 +5,7 @@ import json from .common import POLICIES, PRIVATE, REGISTRATION, RUNTIME, STANDIN, TENANT, assert_denial, require +from .collection_delete_probe import stable_object def policies_ready(h): @@ -46,6 +47,27 @@ def reserved_source_probe(): "sandbox": {"isolation": "standard"}}} +def rc_update_preview(h, path, original, candidate): + current = h.api("GET", path, status=200).json() + require(stable_object(current) == stable_object(original) + and not current["metadata"].get("deletionTimestamp"), + "ReplicationController update target changed beyond status") + for attempt in range(3): + body = copy.deepcopy(candidate) + body["metadata"] = copy.deepcopy(current["metadata"]) + response = h.api("PUT", path + "?dryRun=All", body=body, user="tenant") + if response.status_code != 409 or response.json().get("reason") != "Conflict" or attempt == 2: + return response + refreshed = h.api("GET", path, status=200).json() + require(stable_object(refreshed) == stable_object(current) + and not refreshed["metadata"].get("deletionTimestamp"), + "ReplicationController update conflict changed more than status; no retry") + if refreshed["metadata"]["resourceVersion"] == current["metadata"]["resourceVersion"]: + return response + current = refreshed + raise AssertionError("ReplicationController preview exceeded its bounded attempts") + + def replication_controller_cases(h): collection = f"/api/v1/namespaces/{RUNTIME}/replicationcontrollers" for resource, verb, namespace, subresource, expected in [ @@ -117,10 +139,14 @@ def replication_controller_cases(h): body = copy.deepcopy(candidate) if method == "POST": body["metadata"] = {"name": "e2e-rc-dry-run", "namespace": RUNTIME} - response = h.api(method, target + "?dryRun=All", body=body, user="tenant") + response = (rc_update_preview(h, target, current, body) if method == "PUT" + else h.api(method, target + "?dryRun=All", body=body, user="tenant")) + print("SRE-RC-ADMISSION " + json.dumps({ + "variant": variant, "method": method, "httpStatus": response.status_code, + }), flush=True) if variant == "ordinary": require(response.status_code == (201 if method == "POST" else 200), - "Ordinary ReplicationController request was not admitted") + f"Ordinary ReplicationController {method} was not admitted: HTTP {response.status_code}") else: assert_denial(response, f"{method} ReplicationController {variant}", "kars-sre-private-workloads") diff --git a/tests/e2e/sre_authority/replication_controller_test.py b/tests/e2e/sre_authority/replication_controller_test.py index 13cad2395..91eaeb98f 100644 --- a/tests/e2e/sre_authority/replication_controller_test.py +++ b/tests/e2e/sre_authority/replication_controller_test.py @@ -5,8 +5,10 @@ import copy import unittest +from types import SimpleNamespace +from unittest.mock import Mock -from sre_authority.admission import replication_controller_cases +from sre_authority.admission import rc_update_preview, replication_controller_cases from sre_authority.common import PRIVATE, RUNTIME @@ -20,7 +22,8 @@ def json(self): class Harness: - def __init__(self, *, allow_private=False, wrong_denial=False, conflict=False, replace=False): + def __init__(self, *, allow_private=False, wrong_denial=False, conflict=False, replace=False, + update_conflict=False): self.calls = [] self.object = None self.completed = [] @@ -28,6 +31,7 @@ def __init__(self, *, allow_private=False, wrong_denial=False, conflict=False, r self.wrong_denial = wrong_denial self.conflict = conflict self.replace = replace + self.update_conflict = update_conflict def passed(self, message): self.completed.append(message) @@ -59,6 +63,10 @@ def api(self, method, path, *, body=None, user="admin", status=None): self.object = None response = Response(200, {}) elif "?dryRun=All" in path: + if method == "PUT" and self.update_conflict: + self.update_conflict = False + self.object["metadata"]["resourceVersion"] = "2" + return Response(409, {"kind": "Status", "reason": "Conflict"}) if "template" not in body["spec"]: response = Response(422, {"kind": "Status", "reason": "Invalid", "details": {"causes": [ {"field": "spec.template", "reason": "FieldValueRequired"}, @@ -132,6 +140,42 @@ def test_cleanup_conflict_uses_a_fresh_version_without_dropping_uid_fence(self): {"uid": "owned-rc", "resourceVersion": "2"}, ]) + def test_update_status_conflict_retries_with_the_same_uid_and_fresh_version(self): + harness = Harness(update_conflict=True) + replication_controller_cases(harness) + updates = [body for method, _, body, _ in harness.calls if method == "PUT"] + self.assertEqual(updates[0]["metadata"]["uid"], "owned-rc") + self.assertEqual(updates[1]["metadata"]["uid"], "owned-rc") + self.assertEqual(updates[0]["metadata"]["resourceVersion"], "1") + self.assertEqual(updates[1]["metadata"]["resourceVersion"], "2") + self.assertEqual(len(harness.completed), 1) + + def test_update_preview_never_retries_identity_or_content_changes(self): + original = {"metadata": {"name": "fixture", "uid": "owned", "resourceVersion": "1"}, + "spec": {"replicas": 0}} + for change in ["uid", "spec", "deletionTimestamp"]: + refreshed = copy.deepcopy(original) + refreshed["metadata"]["resourceVersion"] = "2" + if change == "uid": + refreshed["metadata"]["uid"] = "foreign" + elif change == "spec": + refreshed["spec"]["replicas"] = 1 + else: + refreshed["metadata"]["deletionTimestamp"] = "2026-09-10T12:00:00Z" + api = Mock(side_effect=[Response(200, original), Response(409, {"reason": "Conflict"}), + Response(200, refreshed)]) + with self.assertRaises(AssertionError): + rc_update_preview(SimpleNamespace(api=api), "/fixture", original, original) + self.assertEqual(api.call_count, 3) + + def test_update_preview_returns_other_denials_without_retry(self): + original = {"metadata": {"uid": "owned", "resourceVersion": "1"}, "spec": {}} + for code in [403, 422, 500]: + denial = Response(code, {"reason": "Forbidden"}) + api = Mock(side_effect=[Response(200, original), denial]) + self.assertIs(rc_update_preview(SimpleNamespace(api=api), "/fixture", original, original), denial) + self.assertEqual(api.call_count, 2) + def test_replacement_refuses_update_and_cleanup_of_the_foreign_object(self): harness = Harness(replace=True) with self.assertRaises(AssertionError): From 8e617e3e6916ad9cc2e4ea98f7318535c30eff32 Mon Sep 17 00:00:00 2001 From: pallakatos Date: Thu, 10 Sep 2026 15:31:14 +0200 Subject: [PATCH 62/62] test(e2e): wait for asynchronously created sandbox resources Full native qualification completed the SRE legacy/fresh lifecycle and RC boundary cases, then an immediate smoke lookup raced NetworkPolicy creation after the namespace appeared. Reuse the existing bounded exact-resource helper for NetworkPolicy and ServiceAccount creation. Preserve failure when absent and all later policy-content assertions; do not replace actual enforcement proof with a wait. The preceding native job remains recorded as 164 passed/1 failed. All 108 Python harness tests and shell syntax checks pass after this test-only correction. Production controller, policy, authority and timeout values are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb3654cd-f1e0-445a-8734-430800af1903 --- .../2026-09-08-sre-authority-prerequisite.md | 12 ++++++ tests/e2e/run.sh | 4 +- .../e2e/sre_authority/smoke_ordering_test.py | 41 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/sre_authority/smoke_ordering_test.py diff --git a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md index 05b846f89..636a3342b 100644 --- a/docs/security-audits/2026-09-08-sre-authority-prerequisite.md +++ b/docs/security-audits/2026-09-08-sre-authority-prerequisite.md @@ -55,6 +55,18 @@ rerun the native tests. No universal security, CNI-enforcement or complete Bridge-readiness claim is made. Any subsequently identified blocking finding reopens this approval; no failed check, timeout or policy is waived. +The full native run at `8618e9f45d71f3223f35788c6233ebcf56ca5587` +([job 102880906111](https://github.com/Azure/kars/actions/runs/34478247382/job/102880906111)) +completed legacy and fresh SRE migration, the RC CREATE/UPDATE denial matrix, +diagnostic compatibility and retirement. Its final result was 164 passed and +one failed: an unrelated smoke assertion read the NetworkPolicy immediately +after namespace creation, before asynchronous reconciliation reached that +resource. The same run subsequently observed its required ingress policy. +Both initial NetworkPolicy/ServiceAccount smoke checks now use the existing +30-second exact-resource wait helper; missing resources still fail, and policy +content checks remain unchanged. This fixture correction requires a new +exact-head run; the prior failed job is not reclassified. + Signed-off-by: pallakatos (maintainer authorization recorded above) Signed-off-by: GitHub Copilot (delegated AI audit, not an independent human) <223556219+Copilot@users.noreply.github.com> diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 8923b9af9..96fe89da6 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -314,7 +314,7 @@ EOF } test_networkpolicy_created() { - if kubectl get networkpolicy -n kars-e2e-test sandbox-policy --no-headers 2>/dev/null | grep -q sandbox-policy; then + if wait_for_resource networkpolicy sandbox-policy kars-e2e-test 30; then pass "NetworkPolicy created in sandbox namespace" else fail "NetworkPolicy not found" @@ -322,7 +322,7 @@ test_networkpolicy_created() { } test_serviceaccount_created() { - if kubectl get serviceaccount -n kars-e2e-test sandbox --no-headers 2>/dev/null | grep -q sandbox; then + if wait_for_resource serviceaccount sandbox kars-e2e-test 30; then pass "ServiceAccount created in sandbox namespace" else fail "ServiceAccount not found" diff --git a/tests/e2e/sre_authority/smoke_ordering_test.py b/tests/e2e/sre_authority/smoke_ordering_test.py new file mode 100644 index 000000000..5ff018731 --- /dev/null +++ b/tests/e2e/sre_authority/smoke_ordering_test.py @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Smoke assertions wait for their own asynchronous controller resources.""" + +from pathlib import Path +import re +import subprocess +import unittest + + +class SmokeOrderingTests(unittest.TestCase): + def test_resource_checks_use_bounded_exact_waits_and_still_fail_when_absent(self): + source = (Path(__file__).resolve().parents[1] / "run.sh").read_text() + for function, kind, name in [ + ("test_networkpolicy_created", "networkpolicy", "sandbox-policy"), + ("test_serviceaccount_created", "serviceaccount", "sandbox"), + ]: + match = re.search(rf"^{function}\(\) \{{\n.*?^\}}", source, re.MULTILINE | re.DOTALL) + self.assertIsNotNone(match) + for available in [True, False]: + with self.subTest(function=function, available=available): + script = f""" +wait_for_resource() {{ + printf 'WAIT %s %s %s %s\\n' "$1" "$2" "$3" "$4" + return {0 if available else 1} +}} +pass() {{ printf 'PASS %s\\n' "$1"; }} +fail() {{ printf 'FAIL %s\\n' "$1"; return 1; }} +{match.group(0)} +{function} +""" + result = subprocess.run(["bash", "-c", script], text=True, capture_output=True, + timeout=5, check=False) + self.assertEqual(result.returncode, 0 if available else 1) + self.assertIn(f"WAIT {kind} {name} kars-e2e-test 30\n", result.stdout) + self.assertIn("PASS " if available else "FAIL ", result.stdout) + + +if __name__ == "__main__": + unittest.main()