feat(sre): register privileged authority and isolate Kubernetes credentials

.github/workflows/ci.yml

@@ -105,6 +105,9 @@ jobs:
         # transitive oci-client→reqwest) compiles natively with the
         # runner's gcc.
         run: cargo build --release --workspace
+      - name: Legacy Hermes HTTPS client test dependency
+        if: needs.changes.outputs.code == 'true'
+        run: python3 -m pip install 'httpx==0.28.1'
       - name: cargo nextest run (release)
         if: needs.changes.outputs.code == 'true'
         # Reuses the target/release/ artefacts the previous step just
@@ -393,6 +396,67 @@ jobs:
       - name: RBAC idempotency gate
         run: python3 ci/bicep-rbac-idempotency.py
 
+  sre-crd-schema:
+    name: SRE CRD API Schema
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    env:
+      KUBECONFIG: ${{ github.workspace }}/.e2e-sre-schema-kubeconfig
+      PYTHONDONTWRITEBYTECODE: "1"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      # Same pinned tools and unmodified node configuration as e2e-kind.
+      # Kind v0.24.0 defaults to the observed kindest/node:v1.31.0 image.
+      - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
+        with:
+          install_only: true
+          version: v0.24.0
+      - uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4
+        with:
+          version: v1.30.5
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+      - name: Check public-schema diagnostic privacy
+        run: PYTHONPATH=tests/e2e python3 -m unittest sre_authority.registration_schema_test sre_authority.bootstrap_probe_test sre_authority.binding_probe_test sre_authority.legacy_crds_test
+      - name: Create the same disposable API server as the real harness
+        run: kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG"
+      - name: Prove native historical Helm wait orders CRDs before hooks and permits schema upgrades
+        run: PYTHONPATH=tests/e2e python3 -m sre_authority.legacy_crd_probe
+      - name: Reset disposable cluster after historical Helm proof
+        run: |
+          kind delete cluster --name kars-e2e
+          kind create cluster --name kars-e2e --config tests/e2e/kind-config.yaml --kubeconfig "$KUBECONFIG"
+      - name: Validate the SRE CRD against the actual API server
+        id: sre_schema
+        run: python3 tests/e2e/sre_authority/registration_schema.py --exercise
+      - name: Prove controller Pod admission with all chart policies and no image execution
+        id: sre_bootstrap
+        run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --retirement-bind-proof
+      - name: Collect nil-schema adapter candidate evidence without weakening production gates
+        if: failure() && steps.sre_bootstrap.outcome == 'failure'
+        run: PYTHONPATH=tests/e2e python3 -m sre_authority.bootstrap_probe --json-params-candidate
+      - name: Collect namespace-accessor candidate evidence without relaxing the failing production gate
+        if: failure() && steps.sre_schema.outcome == 'failure'
+        run: python3 tests/e2e/sre_authority/registration_schema.py --namespace-accessor-candidate --exercise
+      - name: Upload only public CRD schema evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
+        with:
+          name: sre-crd-schema-${{ github.run_id }}
+          path: |
+            e2e-sre-schema-diag/versions.json
+            e2e-sre-schema-diag/legacy-helm-readiness.json
+            e2e-sre-schema-diag/validation.json
+            e2e-sre-schema-diag/validation-instances.json
+            e2e-sre-schema-diag/namespace-accessor-candidate.json
+            e2e-sre-schema-diag/namespace-accessor-candidate-instances.json
+            e2e-sre-schema-diag/bootstrap-*.json
+            e2e-sre-schema-diag/candidate-bootstrap-*.json
+          if-no-files-found: warn
+          retention-days: 7
+      - name: Remove disposable schema cluster
+        if: always()
+        run: kind delete cluster --name kars-e2e
+
   helm-lint:
     name: Helm Lint
     runs-on: ubuntu-latest
@@ -572,7 +636,7 @@ jobs:
           # Fetch enough history to diff.
           git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true
           if git diff --name-only "$base" "$head" 2>/dev/null \
-             | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then
+             | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|shared/|runtimes/hermes/src/kars_runtime_hermes/plugin/sre|cli/src/(commands/sre|lib/sre|lib/namespace-ownership)|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then
             echo "run=true" >> "$GITHUB_OUTPUT"
           else
             echo "run=false" >> "$GITHUB_OUTPUT"
@@ -616,6 +680,21 @@ jobs:
           docker system prune -af --volumes >/dev/null 2>&1 || true
           df -h
 
+      - name: Set up Node for real SRE authority CLI acceptance
+        if: steps.paths.outputs.run == 'true'
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: cli/package-lock.json
+      - name: Build locked CLI for Kind acceptance
+        if: steps.paths.outputs.run == 'true'
+        working-directory: cli
+        run: npm ci && npm run build
+      - name: SRE Kind unchanged Hermes HTTPS client dependency
+        if: steps.paths.outputs.run == 'true'
+        run: python3 -m pip install 'httpx==0.28.1'
+
       # Pre-built binaries from build-rust are COPY'd into the
       # distroless runtime images; no `cargo build` runs inside Docker.
       # Result: each image build is ~30s instead of ~8min.
@@ -690,7 +769,12 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v4
         with:
           name: e2e-diagnostics-${{ github.run_id }}
-          path: e2e-diag/
+          path: |
+            e2e-diag/
+            e2e-sre-schema-diag/versions.json
+            e2e-sre-schema-diag/validation.json
+            e2e-sre-schema-diag/registration-create.json
+            e2e-sre-schema-diag/bootstrap-*.json
           retention-days: 7
 
   bench-regression:

Cargo.toml

@@ -47,6 +47,10 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"
 # with "Could not automatically determine the process-level
 # CryptoProvider" when multiple feature flags resolve.
 rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs"] }
+rcgen = { version = "0.13.2", default-features = false, features = ["aws_lc_rs", "pem"] }
+tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs"] }
+rustls-pemfile = "2"
+time = "0.3"
 tower = { version = "0.5", features = ["limit"] }
 tower-http = { version = "0.6", features = ["trace", "cors"] }
 hyper = "1"

ci/no-custom-crypto.sh

@@ -67,6 +67,7 @@ ALLOW_PATHS=(
 
 # Production paths to scan.
 PROD_PATHS=(
+  'shared/'
   'controller/src/'
   'inference-router/src/'
   'cli/src/'

ci/no-stubs.sh

@@ -19,6 +19,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)"
 cd "$REPO_ROOT"
 
 PROD_PATHS=(
+  'shared/'
   'controller/src/'
   'inference-router/src/'
   'cli/src/'

ci/security-audit-required.sh

@@ -17,7 +17,7 @@ REPO_ROOT="$(git rev-parse --show-toplevel)"
 cd "$REPO_ROOT"
 
 # Capability-introducing paths — mirrors §4.4 of the plan.
-CAP_RE='^(controller/src/(crd|reconcilers|admission)|inference-router/src/(mcp|a2a|providers|routes)|cli/src/(commands|migrate|adapters)|runtimes/openclaw/src/(core|index\.ts)|sandbox-images/[^/]+/(Dockerfile|entrypoint\.sh)|cli/profiles/|deploy/seccomp/|deploy/helm/kars/files/)'
+CAP_RE='^(controller/src/(crd|reconcilers|admission)|inference-router/src/(mcp|a2a|providers|routes)|cli/src/(commands|migrate|adapters)|runtimes/openclaw/src/(core|index\.ts)|sandbox-images/[^/]+/(Dockerfile|entrypoint\.sh)|cli/profiles/|deploy/seccomp/|deploy/helm/kars/files/|shared/.*\.rs$)'
 
 changed=$(git diff --name-only "${BASE_REF}...HEAD" 2>/dev/null || git diff --name-only HEAD)
 # Exclude test files — they exercise capabilities but don't introduce

cli/src/commands/destroy.ts

@@ -4,6 +4,7 @@
 import { Command } from "commander";
 import chalk from "chalk";
 import ora from "ora";
+import { assertDestroySafe } from "../lib/sre-authority.js";
 
 export function destroyCommand(): Command {
   const cmd = new Command("destroy");
@@ -22,6 +23,11 @@ export function destroyCommand(): Command {
       const rg = options.resourceGroup || `kars-${options.region}`;
       // Propagate --context to every kubectl invocation in this command.
       const kctlCtx = options.context ? ["--context", options.context] : [];
+      if ((!options.local || options.cloud) && (!name || name === "sre" || options.all)) {
+        const { execa } = await import("execa");
+        await assertDestroySafe((file,args,commandOptions) =>
+          execa(file,[...kctlCtx,...args],commandOptions));
+      }
 
       if (options.all) {
         // Full teardown — delete the entire resource group

cli/src/commands/dev/local-k8s.ts

@@ -31,6 +31,7 @@ import { ensureAgtRepo, ensureAgtWheels } from "../../lib/agt-bootstrap.js";
 import { resolveBundledAsset, requireBundledAsset, findRepoRootOrNull } from "../../lib/repo-assets.js";
 import { buildCopilotFallbackChain } from "../../github-copilot.js";
 import { CLAIM, prepareCredentialNamespace } from "../../lib/namespace-ownership.js";
+import { assertSafeMutation } from "../../lib/sre-authority.js";
 
 export interface LocalK8sOptions {
   /** Sandbox / agent name. Reused as Helm release name suffix. */
@@ -1337,6 +1338,10 @@ export async function runLocalK8s(opts: LocalK8sOptions): Promise<void> {
 
   stepper.step(`Ensuring kind cluster '${opts.clusterName}' exists…`);
   await ensureCluster(tools.kind, opts.clusterName, tools.env);
+  await assertSafeMutation((file,args,commandOptions) => execa(
+    file === "kubectl" ? tools.kubectl : file,
+    ["--context", `kind-${opts.clusterName}`, ...args], commandOptions,
+  ));
   stepper.done(`kind cluster '${opts.clusterName}' is ready`);
 
   // Ensure the three local-dev images exist AND match the host arch.

cli/src/commands/push-apply.test.ts

@@ -146,6 +146,14 @@ function fixture(options: { legacyCore?: boolean; mesh?: "absent" | "helm" | "ex
 }
 
 describe("selected core push artifacts", () => {
+  it("still requires SRE authority preflight before resolving or applying valid core artifacts", async () => {
+    const f = fixture({ fail: "karssreregistrations.kars.azure.com" });
+    await expect(applyPushedImages(f.execute, [pushed("controller")], "chart"))
+      .rejects.toThrow("karssreregistrations.kars.azure.com");
+    expect(f.calls().some(([bin, args]) => bin === "az"
+      || ["upgrade", "patch", "annotate", "rollout"].includes(args[0]))).toBe(false);
+  });
+
   it("moves a GHCR/pinned Helm controller to its pushed ACR digest without resetting customer values", async () => {
     const f = fixture();
     await applyPushedImages(f.execute, [pushed("controller")], "chart");

cli/src/commands/push-apply.ts

@@ -11,6 +11,7 @@ import {
 import { coreImageValues, imageValueArgs, PUSH_COMPONENTS, resolvePushedArtifacts, type PushedImage } from "../lib/image-targets.js";
 import { inspectCoreInstallation, recheckCoreOwnership, requireHealthyDeployment, updateLegacyCore, verifyCoreConfiguration } from "../lib/core-image-apply.js";
 import { inspectSandboxPlans, refreshSandboxImages } from "../lib/sandbox-image-apply.js";
+import { assertSafeMutation } from "../lib/sre-authority.js";
 
 export interface PushApplyResult {
   applied: string[];
@@ -41,6 +42,7 @@ export async function applyPushedImages(
     throw new Error("External or absent AgentMesh cannot be updated; choose explicit core targets instead.");
   }
   if (core?.kind === "helm" && mesh) assertMeshReleaseConsistency(mesh, core.values);
+  await assertSafeMutation(execute);
   const artifacts = await resolvePushedArtifacts(execute, deployable);
   const coreImages = artifacts.filter(item => !isMesh(item));
   const meshImages: MeshImages = {};

cli/src/commands/sre-authority.ts

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { Command } from "commander";
+import { execa } from "execa";
+import { requireBundledAsset } from "../lib/repo-assets.js";
+import { enroll, preview, requireRegistrar, retire, waitForAuthority, type Execute } from "../lib/sre-authority.js";
+import { stageSource } from "../lib/sre-source.js";
+import { stageAuthority } from "../lib/sre-stage.js";
+
+function executor(context?: string): Execute {
+  return (file, args, options) => execa(file, [
+    ...(context ? [file === "helm" ? "--kube-context" : "--context", context] : []), ...args,
+  ], options);
+}
+
+export function authorityCommand(): Command {
+  const command = new Command("authority").description("Stage, review, enroll and retire cluster-authorized SRE privacy");
+  const common = (name: string) => command.command(name)
+    .option("--namespace <namespace>", "Controller/release namespace", "kars-system")
+    .option("--release <release>", "Owning Helm release", "kars")
+    .option("--context <context>", "Kubernetes context");
+  common("stage")
+    .description("Explicit registrar staging: install authority APIs/controller while retaining legacy grants unchanged")
+    .requiredOption("--controller-image <image>", "Qualified prerequisite controller repository:tag")
+    .requiredOption("--router-image <image>", "Qualified prerequisite router repository:tag")
+    .option("--dry-run", "Server-side preview without deployment changes")
+    .action(async options => {
+      const execute = executor(options.context);
+      await stageAuthority(execute,requireBundledAsset("deploy/helm/kars"),options.namespace,options.release,
+        options.controllerImage,options.routerImage,!!options.dryRun);
+      console.log("Authority controller staged. Preview and explicitly enroll the exact SRE source/grants before normal upgrades.");
+    });
+  common("preview").description("Read exact enrollment identities and legacy grants; no mutations")
+    .action(async options => {
+      const spec = await preview(executor(options.context),options.namespace,options.release);
+      console.log(JSON.stringify(spec,null,2));
+      for (const binding of spec.legacyBindings) {
+        console.log(`--binding '${binding.kind}/${binding.namespace ?? ""}/${binding.name}=${binding.uid}@${binding.resourceVersion}'`);
+      }
+      if (spec.legacyConsumer) console.log(`--consumer '${spec.legacyConsumer.uid}@${spec.legacyConsumer.resourceVersion}'`);
+    });
+  common("stage-source").description("Atomically create a genuinely new, unprivileged SRE source and wait for its exact claim")
+    .option("--model <model>", "SRE model deployment")
+    .action(async options => {
+      const execute=executor(options.context);
+      await requireRegistrar(execute);
+      const result=await execute("helm",["template",options.release,requireBundledAsset("deploy/helm/kars"),
+        "--namespace",options.namespace,"--show-only","templates/sre.yaml",
+        "--set","sre.enabled=true","--set","azure.workloadIdentity.clientId=dummy",
+        ...(options.model?["--set-string",`sre.model=${options.model}`]:[])],{stdio:"pipe"});
+      const created=await stageSource(execute,result.stdout,options.namespace,options.release);
+      console.log(`Created and claimed source: --sandbox-uid ${created.uid} --namespace-uid ${created.namespaceUid}`);
+    });
+  common("enroll").description("Enroll only reviewed source/namespace and grant UIDs under cluster registrar authority")
+    .requiredOption("--sandbox-uid <uid>", "Reviewed canonical Sandbox UID")
+    .requiredOption("--namespace-uid <uid>", "Reviewed claimed runtime namespace UID")
+    .option("--binding <review>", "Exact binding review from preview; repeat per binding", (value: string, all: string[]) => [...all,value], [])
+    .option("--consumer <uid@resourceVersion>", "Reviewed legacy SRE Deployment")
+    .option("--registration-uid <uid>", "Required when updating an existing registration")
+    .option("--resource-version <version>", "Required when updating an existing registration")
+    .option("--dry-run", "Print the enrollment without writing it")
+    .action(async options => {
+      const execute = executor(options.context);
+      const spec = await preview(execute,options.namespace,options.release);
+      console.log(await enroll(execute,spec,options,!!options.dryRun));
+    });
+  common("migrate").description("Wait for the controller to complete the explicitly enrolled migration")
+    .action(async options => {
+      await requireRegistrar(executor(options.context));
+      await waitForAuthority(executor(options.context),"Ready");
+      console.log("SRE authority Ready: legacy credentials denied and private renewable identity configured.");
+    });
+  common("retire").description("Disable and retire a reviewed registration before SRE uninstall")
+    .requiredOption("--registration-uid <uid>", "Reviewed registration UID")
+    .requiredOption("--resource-version <version>", "Reviewed registration resourceVersion")
+    .action(async options => {
+      await retire(executor(options.context),options.registrationUid,options.resourceVersion);
+      console.log("SRE authority Retired; owned private grants and credentials are revoked.");
+    });
+  return command;
+}