feat(inference): add compatible local provider failover

.github/workflows/ci.yml

@@ -416,6 +416,17 @@ jobs:
           grep -q 'ghcr.io/azure/kars-controller:latest' /tmp/kars-existing-aks.yaml
           grep -q 'name: agentmesh-relay' /tmp/kars-existing-aks.yaml
 
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+      - name: Restore existing chart-test runner
+        run: npm ci --prefix cli
+      - name: Local inference legacy-values regression
+        run: |
+          node cli/node_modules/vitest/vitest.mjs run \
+            --config "$GITHUB_WORKSPACE/cli/vitest.config.ts" \
+            --root deploy/helm/kars/tests
+
   security-scan:
     name: Security Scan
     runs-on: ubuntu-latest

controller/src/inference_policy_reconciler.rs

@@ -743,7 +743,10 @@ async fn ensure_profile_configmap(
                     "app.kubernetes.io/managed-by".into(),
                     "kars-controller".into(),
                 ),
-                ("kars.azure.com/inferencepolicy".into(), owner.into()),
+                (
+                    "kars.azure.com/inferencepolicy".into(),
+                    crate::labels::value(owner),
+                ),
                 (
                     "kars.azure.com/artifact".into(),
                     "inference-policy-profile".into(),

controller/src/kars_receipt_launch.rs

@@ -160,6 +160,7 @@ mod tests {
                     }],
                     isolation: Some("enhanced".into()),
                     memory: Some("review-memory".into()),
+                    model_fallbacks: Vec::new(),
                 }),
                 display_name: Some("Review".into()),
             },

controller/src/kars_task.rs

@@ -182,6 +182,11 @@ pub struct TaskBlueprint {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub model: Option<TaskModel>,
 
+    /// Ordered alternative inference routes; absent preserves the default route.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[schemars(schema_with = "crate::task_models::fallback_schema")]
+    pub model_fallbacks: Vec<TaskModel>,
+
     /// System prompt / standing instructions for the agent, in addition to the
     /// objective. Drives `KarsSandbox.spec.agent.instructions`.
     #[serde(default, skip_serializing_if = "Option::is_none")]

controller/src/kars_task_authorization_tests.rs

@@ -36,6 +36,7 @@ fn spec() -> KarsTaskSpec {
             }],
             isolation: Some("standard".into()),
             memory: Some("team-memory".into()),
+            model_fallbacks: Vec::new(),
         }),
         ..Default::default()
     }

controller/src/kars_task_execution.rs

@@ -22,7 +22,7 @@
 use kube::api::{Api, DeleteParams, DynamicObject, ObjectMeta, PostParams, Preconditions};
 use kube::core::ApiResource;
 use kube::{Client, ResourceExt};
-use serde_json::json;
+use serde_json::{Value, json};
 
 use crate::kars_task::{KarsTask, TaskBlueprint, TaskEnvelope};
 
@@ -107,6 +107,23 @@ fn network_policy(blueprint: &TaskBlueprint) -> serde_json::Value {
     })
 }
 
+/// Build primary and fallback routes from the shared normalized blueprint.
+fn inference_spec(task_name: &str, blueprint: &TaskBlueprint) -> Result<Value, kube::Error> {
+    let primary = blueprint
+        .model
+        .as_ref()
+        .ok_or_else(|| contract_error("effective task model is missing".into()))?;
+    Ok(json!({
+        "appliesTo": { "sandboxName": task_name },
+        "modelPreference": {
+            "primary": primary,
+            "fallback": crate::task_models::fallback_routes(
+                &blueprint.model_fallbacks, &primary.provider, &primary.deployment,
+            ),
+        },
+    }))
+}
+
 /// Materialize the InferencePolicy + KarsSandbox for a launched task using
 /// atomic creation or version-checked owned updates, then read sandbox status.
 pub async fn materialize(
@@ -123,12 +140,7 @@ pub async fn materialize(
 
     // 1. InferencePolicy scoped to this sandbox. Model: blueprint wins, else
     //    the controller default (required — without it the sandbox degrades).
-    let inference_spec = json!({
-        "appliesTo": { "sandboxName": task_name },
-        "modelPreference": {
-            "primary": blueprint.model,
-        },
-    });
+    let inference_spec = inference_spec(&task_name, &blueprint)?;
     apply_dynamic(
         client,
         namespace,

controller/src/kars_task_execution_tests.rs

@@ -5,6 +5,44 @@ use super::*;
 use wiremock::matchers::{method, path};
 use wiremock::{Mock, MockServer, ResponseTemplate};
 
+#[test]
+fn materialization_keeps_fallbacks_from_the_shared_effective_blueprint() {
+    use crate::kars_task::{KarsTaskSpec, TaskModel, blueprint::effective_blueprint_with_model};
+    let primary = TaskModel {
+        provider: "azure-openai".into(),
+        deployment: "primary".into(),
+    };
+    let spec = KarsTaskSpec {
+        blueprint: Some(TaskBlueprint {
+            model_fallbacks: vec![
+                primary.clone(),
+                TaskModel {
+                    provider: "backup".into(),
+                    deployment: "second".into(),
+                },
+                TaskModel {
+                    provider: "backup".into(),
+                    deployment: "second".into(),
+                },
+            ],
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    let effective = effective_blueprint_with_model(&spec, &primary);
+    let policy = inference_spec("demo", &effective).unwrap();
+    assert_eq!(
+        policy["modelPreference"]["primary"],
+        serde_json::to_value(primary).unwrap()
+    );
+    assert_eq!(
+        policy["modelPreference"]["fallback"],
+        json!([
+            {"provider":"backup","deployment":"second"},
+        ])
+    );
+}
+
 const OBJECT_PATH: &str = "/apis/kars.azure.com/v1alpha1/namespaces/default/karssandboxes/demo";
 const COLLECTION_PATH: &str = "/apis/kars.azure.com/v1alpha1/namespaces/default/karssandboxes";
 
@@ -210,6 +248,10 @@ async fn materialized_resources_match_the_authorization_blueprint() {
             deployment: "reviewed-model".into(),
             provider: String::new(),
         }),
+        model_fallbacks: vec![TaskModel {
+            provider: "backup-account".into(),
+            deployment: "fallback-model".into(),
+        }],
         instructions: Some(" Cite evidence. ".into()),
         tool_policy: Some("read-only".into()),
         mcp_servers: vec!["docs".into()],
@@ -264,6 +306,10 @@ async fn materialized_resources_match_the_authorization_blueprint() {
         specs[0]["modelPreference"]["primary"],
         json!(effective.model)
     );
+    assert_eq!(
+        specs[0]["modelPreference"]["fallback"],
+        json!(effective.model_fallbacks)
+    );
     assert_eq!(specs[1]["runtime"]["kind"], "MicrosoftAgentFramework");
     assert_eq!(specs[1]["sandbox"]["isolation"], json!(effective.isolation));
     assert_eq!(

controller/src/kars_team_reconciler/tests.rs

@@ -9,6 +9,55 @@ use crate::kars_task::{
 use crate::kars_team::{KarsTeamSpec, TeamCadence, TeamRole};
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
 
+#[test]
+fn inherited_fallbacks_reach_principal_member_and_run_authority_without_weakening_budget_gate() {
+    let mut team = team();
+    let primary = TaskModel {
+        provider: "azure-openai".into(),
+        deployment: "primary".into(),
+    };
+    team.spec.blueprint = Some(TaskBlueprint {
+        model: Some(primary.clone()),
+        model_fallbacks: vec![TaskModel {
+            provider: "backup".into(),
+            deployment: "secondary".into(),
+        }],
+        ..Default::default()
+    });
+    assert!(
+        crate::kars_task::validate_execution_contract(&specs::run_spec(&team, "").unwrap())
+            .unwrap_err()
+            .contains("UnsupportedLaunchBudget")
+    );
+    team.spec.envelope.budget = None;
+    let role = TeamRole {
+        name: "reviewer".into(),
+        ..Default::default()
+    };
+    for spec in [
+        specs::principal_spec(&team),
+        specs::member_spec(&team, &role),
+        specs::run_spec(&team, "").unwrap(),
+    ] {
+        let effective =
+            crate::kars_task::blueprint::effective_blueprint_with_model(&spec, &primary);
+        assert_eq!(effective.model_fallbacks.len(), 1);
+        assert_eq!(effective.model_fallbacks[0].provider, "backup");
+        let mut changed = spec.clone();
+        changed.blueprint.as_mut().unwrap().model_fallbacks[0].deployment = "changed".into();
+        assert_ne!(
+            spec.authorization_digest_with_model(&primary),
+            changed.authorization_digest_with_model(&primary),
+        );
+    }
+    let explicit = TeamRole {
+        blueprint: Some(TaskBlueprint::default()),
+        ..role
+    };
+    let member = specs::member_spec(&team, &explicit);
+    assert!(member.blueprint.unwrap().model_fallbacks.is_empty());
+}
+
 pub(super) fn team() -> KarsTeam {
     let mut team = KarsTeam::new(
         "eng",

controller/src/labels.rs

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/// Keep short label values readable and long/non-ASCII resource identities stable.
+pub(crate) fn value(input: &str) -> String {
+    let valid = input.len() <= 63
+        && input
+            .bytes()
+            .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'-' | b'_' | b'.'))
+        && input
+            .as_bytes()
+            .first()
+            .is_some_and(u8::is_ascii_alphanumeric)
+        && input
+            .as_bytes()
+            .last()
+            .is_some_and(u8::is_ascii_alphanumeric);
+    if valid {
+        return input.to_string();
+    }
+    format!(
+        "id-{}",
+        crate::kars_receipt_log::sha256_hex(input.as_bytes())
+    )
+    .chars()
+    .take(63)
+    .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn resource_names_fit_label_values_without_colliding() {
+        assert_eq!(value("short-name"), "short-name");
+        let a = format!("{}a", "long".repeat(35));
+        let b = format!("{}b", "long".repeat(35));
+        assert!(value(&a).len() <= 63);
+        assert_ne!(value(&a), value(&b));
+        assert_eq!(value(&a), value(&a));
+        assert!(value("資料").is_ascii());
+    }
+}

controller/src/main.rs

@@ -58,6 +58,7 @@ mod kars_task_execution;
 mod kars_task_reconciler;
 mod kars_team;
 mod kars_team_reconciler;
+mod labels;
 mod leader_election;
 mod mcp_server;
 mod mcp_server_reconciler;
@@ -72,6 +73,7 @@ mod providers;
 mod reconciler;
 mod signer_policy;
 mod status;
+mod task_models;
 mod team_commons;
 mod team_digest;
 #[allow(dead_code)] // helpers consumed by tool_policy_reconciler + future slices.