From c43f7c1c4a63f2696a5ffcc37d35609201e5cc38 Mon Sep 17 00:00:00 2001 From: Benny Zlotnik Date: Tue, 8 Sep 2026 08:19:07 +0300 Subject: [PATCH 1/3] feat(cuttlefish): support managed Pod lifecycle Restrict managed creation to one CVD and the provisioned configuration so requests cannot exceed the Pod resource budget. Serialize lifecycle operations and track guest intent plus the original runtime ID for health checks, including warm Pods before the first lease. Signed-off-by: Benny Zlotnik --- .../jumpstarter_driver_cuttlefish/driver.py | 99 ++++++++++++++++- .../driver_test.py | 102 ++++++++++++++++++ .../jumpstarter_driver_cuttlefish/health.py | 66 ++++++++++++ .../health_test.py | 99 +++++++++++++++++ 4 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py create mode 100644 python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py index 42ec00deb..e92037121 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py @@ -1,8 +1,11 @@ import json +import os import subprocess +import threading import time from collections.abc import Generator from dataclasses import dataclass, field +from pathlib import Path import requests from jumpstarter_driver_adb.driver import AdbServer @@ -39,16 +42,44 @@ class Cuttlefish(Driver): boot_timeout: int = 300 env_config: dict = field(default_factory=dict) webrtc_url: str = "" + managed: bool = False + health_state_path: str = "" + runtime_id_path: str = "" + health_ports: list[int] = field(default_factory=list) + _operation_lock: threading.RLock = field(default_factory=threading.RLock, init=False, repr=False) + _health: dict = field(default_factory=dict, init=False, repr=False) _cvd_group: str | None = field(default=None, init=False, repr=False) _cvd_name: str | None = field(default=None, init=False, repr=False) def __post_init__(self): if hasattr(super(), "__post_init__"): super().__post_init__() + if self.managed: + self._validate_managed_config() + self._health = json.loads(Path(self.health_state_path).read_text()) + if self._health["runtime_id"] != Path(self.runtime_id_path).read_text().strip(): + raise CuttlefishError("Cuttlefish runtime restarted before driver initialization") + self._health.update(url=self._base_url, ports=self.health_ports) + self._write_health() + self.children["power"] = CvdPower(parent=self) self.children["storage"] = CvdFlasher(parent=self) self.children["adb"] = AdbServer(host="127.0.0.1", port=self.adb_server_port) + def _validate_managed_config(self): + instances = self.env_config.get("instances", []) + if len(instances) != 1 or not isinstance(instances[0], dict): + raise CuttlefishError("managed Cuttlefish requires exactly one instance") + if not self.health_state_path or not self.runtime_id_path: + raise CuttlefishError("managed Cuttlefish requires health and runtime ID paths") + + def _write_health(self): + if not self.managed: + return + temporary = Path(self.health_state_path + ".tmp") + temporary.write_text(json.dumps(self._health)) + os.replace(temporary, self.health_state_path) + @classmethod def client(cls) -> str: return "jumpstarter_driver_cuttlefish.client.CuttlefishClient" @@ -125,7 +156,51 @@ def _wait_for_operation(self, op_name: str, timeout: float = 300) -> dict: return r.json() raise CuttlefishTimeout(f"operation {op_name} timed out after {timeout}s") - def _do_operation( + def _validate_creation(self, data): + # Accept only the budgeted, provisioner-approved configuration. Alternate + # HO creation forms and additional groups would bypass the Pod contract. + if data != {"env_config": self.env_config}: + raise CuttlefishError("managed create_cvd requires the configured env_config") + existing = self._request("GET", "/cvds") + if not isinstance(existing, dict) or not isinstance(existing.get("cvds"), list): + raise CuttlefishError("invalid CVD inventory; refusing creation") + if existing["cvds"]: + raise CuttlefishError("managed Cuttlefish already has a CVD; destroy it before creating another") + + def _do_operation(self, method: str, path: str, data: dict | None = None, timeout: float = 300): + if not self.managed: + return self._perform_operation(method, path, data, timeout) + with self._operation_lock: + if method == "POST" and path == "/cvds": + self._validate_creation(data) + state = self._health.get("state", "off") + target_state = state + if path == "/reset" or method == "DELETE" or path.endswith("/:stop"): + target_state = "off" + elif path == "/cvds" or path.endswith(("/:start", "/:restart", "/:powerwash")): + target_state = "running" + # Probes allow bounded transitions, then fail closed if the operation + # hangs or leaves the target's state unknown. + self._health.update(state="transition", deadline=time.monotonic() + timeout + 30) + self._write_health() + try: + result = self._perform_operation(method, path, data, timeout) + except Exception: + self._health["state"] = "failed" + self._write_health() + raise + if isinstance(result, dict) and path == "/cvds": + cvds = result.get("cvds", []) + if len(cvds) == 1: + self._cvd_group = cvds[0].get("group") + self._cvd_name = cvds[0].get("name") + self._health.update( + state=target_state, group=self._cvd_group or self.group, name=self._cvd_name or self.name, + ) + self._write_health() + return result + + def _perform_operation( self, method: str, path: str, @@ -295,7 +370,7 @@ def create_cvd(self, config_json: str) -> str: @export def start_cvd(self) -> str: - return self._fmt(self._do_operation("POST", f"{self._cvd_path}/:start")) + return self._fmt(self._do_operation("POST", f"{self._cvd_path}/:start", {})) @export def stop_cvd(self) -> str: @@ -358,9 +433,16 @@ def client(cls) -> str: return "jumpstarter_driver_cuttlefish.client.CvdPowerClient" @export - def on(self) -> None: # noqa: C901 + def on(self) -> None: + with self.parent._operation_lock: + self._on() + + def _on(self) -> None: # noqa: C901 existing = self.parent._get_existing_cvds() + if self.parent.managed and len(existing) > 1: + raise CuttlefishError("managed Cuttlefish inventory has multiple CVDs") + if len(existing) > 1: self.logger.warning( "Found %d stale CVDs in group %s, deleting", len(existing), self.parent._cvd_group or self.parent.group @@ -392,7 +474,7 @@ def on(self) -> None: # noqa: C901 cvd.get("status"), ) if cvd.get("status") != "Running": - self.parent._do_operation("POST", f"{self.parent._cvd_path}/:start") + self.parent.start_cvd() else: self.logger.info("Creating CVD from env_config") try: @@ -426,12 +508,21 @@ def on(self) -> None: # noqa: C901 ) break + if self.parent.managed: + with self.parent._operation_lock: + self.parent._health.update(state="running", group=self.parent._cvd_group or self.parent.group, + name=self.parent._cvd_name or self.parent.name) + self.parent._write_health() self.parent._auto_connect_adb() if self.parent.boot_timeout: self.parent._wait_boot(self.parent.boot_timeout) @export def off(self, destroy: bool = False) -> None: + with self.parent._operation_lock: + self._off(destroy) + + def _off(self, destroy: bool) -> None: p = self.parent cvd_id = f"{p._cvd_group or p.group}/{p._cvd_name or p.name}" if destroy: diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py index 50207d0dd..4a0458241 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py @@ -149,6 +149,7 @@ def test_start_cvd(requests_mock, drv): _mock_op(requests_mock, "post", "/cvds/cvd_1/dev1/:start") result = json.loads(drv.start_cvd()) assert result["done"] is True + assert requests_mock.request_history[0].json() == {} def test_stop_cvd(requests_mock, drv): @@ -404,6 +405,8 @@ def test_cvd_power_on_existing_stopped(requests_mock, drv): requests_mock.post(f"{BASE}/cvds/cvd_1/dev1/:start", json={"name": "op-1", "done": False}) requests_mock.post(f"{BASE}/operations/op-1/:wait", json={"name": "op-1", "done": True}) power.on() + start_request = next(r for r in requests_mock.request_history if r.url.endswith("/:start")) + assert start_request.json() == {} def test_cvd_power_on_create_new(requests_mock, drv): @@ -517,3 +520,102 @@ def test_cvd_power_on_ignores_other_groups(requests_mock, drv): assert drv._cvd_group == "cvd_1" assert drv._cvd_name == "dev1" assert not any(r.method == "DELETE" for r in requests_mock.request_history) + + +@pytest.fixture +def managed_drv(drv, tmp_path): + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("first-runtime") + drv.managed = True + drv.runtime_id_path = str(runtime_id) + drv.health_state_path = str(tmp_path / "health.json") + drv.env_config = {"instances": [{"vm": {"memory_mb": 8192}}]} + drv._health = {"state": "off", "runtime_id": "first-runtime", "runtime_id_path": str(runtime_id)} + drv._write_health() + return drv + + +def test_managed_rejects_alternate_creation(requests_mock, managed_drv): + for config in ({}, {"env_config": {"instances": [{}, {}]}}, {"cvd": {}}, {"env_config": {}}): + with pytest.raises(CuttlefishError, match="configured env_config"): + managed_drv.create_cvd(json.dumps(config)) + assert requests_mock.call_count == 0 + + +def test_managed_creation_checks_all_groups(requests_mock, managed_drv): + requests_mock.get(f"{BASE}/cvds", json={"cvds": [{"group": "other", "name": "other"}]}) + with pytest.raises(CuttlefishError, match="already has a CVD"): + managed_drv.create_cvd(json.dumps({"env_config": managed_drv.env_config})) + assert all(r.method == "GET" for r in requests_mock.request_history) + + +def test_managed_power_creation_checks_all_groups(requests_mock, managed_drv): + requests_mock.get(f"{BASE}/cvds", json={"cvds": [{"group": "other", "name": "other"}]}) + with pytest.raises(CuttlefishError, match="already has a CVD"): + managed_drv.children["power"].on() + assert all(r.method == "GET" for r in requests_mock.request_history) + + +def test_managed_serializes_creation(managed_drv): + from concurrent.futures import ThreadPoolExecutor + + inventory = [] + + def create(*args): + inventory.append({"group": "cvd_1", "name": "dev1"}) + return {"cvds": inventory} + + def attempt(): + try: + managed_drv.create_cvd(json.dumps({"env_config": managed_drv.env_config})) + return True + except CuttlefishError: + return False + + with patch.object(managed_drv, "_request", side_effect=lambda *args: {"cvds": inventory}), \ + patch.object(managed_drv, "_perform_operation", side_effect=create), ThreadPoolExecutor(2) as pool: + assert sorted(pool.map(lambda _: attempt(), range(2))) == [False, True] + assert len(inventory) == 1 + + +@pytest.mark.parametrize("operation,expected", [ + ("start_cvd", "running"), ("stop_cvd", "off"), ("delete_cvd", "off"), + ("restart_cvd", "running"), ("powerwash_cvd", "running"), ("reset_host", "off"), +]) +def test_managed_records_power_intent(managed_drv, operation, expected): + from pathlib import Path + + def perform(*args): + assert json.loads(Path(managed_drv.health_state_path).read_text())["state"] == "transition" + return {} + + with patch.object(managed_drv, "_perform_operation", side_effect=perform): + getattr(managed_drv, operation)() + assert json.loads(Path(managed_drv.health_state_path).read_text())["state"] == expected + + +def test_managed_records_failed_operation(managed_drv): + from pathlib import Path + + with patch.object(managed_drv, "_perform_operation", side_effect=CuttlefishError("runtime died")): + with pytest.raises(CuttlefishError): + managed_drv.start_cvd() + assert json.loads(Path(managed_drv.health_state_path).read_text())["state"] == "failed" + + +def test_managed_initialization_preserves_startup_runtime_id(drv, tmp_path): + from .health import initialize + + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("runtime-1") + state_path = tmp_path / "health.json" + initialize(str(state_path), str(runtime_id), BASE) + config = {"managed": True, "runtime_id_path": str(runtime_id), "health_state_path": str(state_path), + "env_config": {"instances": [{}]}, "health_ports": [7681]} + instance = Cuttlefish(**config) + assert instance._health["runtime_id"] == "runtime-1" + assert instance._health["ports"] == [7681] + instance.close() + runtime_id.write_text("runtime-2") + with pytest.raises(CuttlefishError, match="restarted"): + Cuttlefish(**config) diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py new file mode 100644 index 000000000..324347d2e --- /dev/null +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py @@ -0,0 +1,66 @@ +"""Pod liveness check using driver intent rather than requiring an always-on guest.""" + +import json +import os +import sys +import time +import urllib.request +from pathlib import Path + + +def initialize(state_path: str, runtime_id_path: str, url: str) -> None: + runtime_id = Path(runtime_id_path).read_text().strip() + if not runtime_id: + raise RuntimeError("Cuttlefish runtime ID is empty") + Path(state_path).write_text(json.dumps({ + "runtime_id_path": runtime_id_path, "runtime_id": runtime_id, + "url": url, "ports": [], "state": "off", + })) + + +def check(state_path: str) -> None: + state = json.loads(Path(state_path).read_text()) + runtime_id = Path(state["runtime_id_path"]).read_text().strip() + if not runtime_id or runtime_id != state["runtime_id"]: + raise RuntimeError("Cuttlefish runtime restarted; release the lease to replace this Pod") + url = state["url"] + with urllib.request.urlopen(f"{url}/_debug/statusz", timeout=2): + pass + if state["state"] == "off": + return + if state["state"] == "transition" and time.monotonic() < state["deadline"]: + return + if state["state"] != "running": + raise RuntimeError("Cuttlefish operation failed or timed out") + with urllib.request.urlopen(f"{url}/cvds", timeout=2) as response: + cvds = json.load(response)["cvds"] + if len(cvds) != 1 or any(cvds[0].get(key) != state[key] for key in ("group", "name")): + raise RuntimeError("Cuttlefish inventory no longer matches this exporter") + if cvds[0].get("status") != "Running": + raise RuntimeError("CVD stopped unexpectedly") + if not set(state["ports"]).issubset(listening_ports()): + raise RuntimeError("Cuttlefish simulator or relay listener is missing") + + +def listening_ports() -> set[int]: + # Opening an HCI connection can create a simulator peer. Inspect the shared + # Pod network namespace instead of disturbing active Bluetooth sessions. + listeners = set() + for table in ("/proc/net/tcp", "/proc/net/tcp6"): + for line in Path(table).read_text().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A": + listeners.add(int(fields[1].rsplit(":", 1)[1], 16)) + return listeners + + +if __name__ == "__main__": + try: + if sys.argv[1] == "--run-exporter": + initialize(sys.argv[2], sys.argv[3], sys.argv[4]) + os.execvp("jmp", ["jmp", "run", "--exporter-config", sys.argv[5]]) + else: + check(sys.argv[1]) + except Exception as exc: + print(f"Cuttlefish unhealthy: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py new file mode 100644 index 000000000..969f6a6ab --- /dev/null +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py @@ -0,0 +1,99 @@ +import io +import json +import time +from pathlib import Path +from unittest.mock import patch + +import pytest + +from .health import check, listening_ports + + +@pytest.fixture +def health_state(tmp_path): + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("runtime-1") + return tmp_path / "health.json", { + "runtime_id_path": str(runtime_id), "runtime_id": "runtime-1", + "url": "http://127.0.0.1:2081", "state": "running", + "group": "cvd", "name": "1", "ports": [7681, 7300, 17681, 17300], + } + + +def run_check(health_state, cvds=None, ports=None): + path, state = health_state + path.write_text(json.dumps(state)) + if cvds is None: + cvds = [{"group": "cvd", "name": "1", "status": "Running"}] + inventory = json.dumps({"cvds": cvds}).encode() + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", + side_effect=lambda *args, **kwargs: io.BytesIO(inventory)), \ + patch("jumpstarter_driver_cuttlefish.health.listening_ports", + return_value=set(state["ports"] if ports is None else ports)): + check(str(path)) + + +def test_running_guest_and_simulators(health_state): + run_check(health_state) + + +@pytest.mark.parametrize("cvds", [[], [{"group": "cvd", "name": "1", "status": "Stopped"}], + [{"group": "other", "name": "1", "status": "Running"}]]) +def test_guest_failure(health_state, cvds): + with pytest.raises(RuntimeError): + run_check(health_state, cvds=cvds) + + +@pytest.mark.parametrize("missing", [7681, 7300, 17681, 17300]) +def test_simulator_or_relay_failure(health_state, missing): + ports = set(health_state[1]["ports"]) - {missing} + with pytest.raises(RuntimeError, match="listener is missing"): + run_check(health_state, ports=ports) + + +def test_intentional_power_off(health_state): + health_state[1]["state"] = "off" + run_check(health_state, cvds=[], ports=[]) + + +def test_bounded_transition(health_state): + health_state[1].update(state="transition", deadline=time.monotonic() + 60) + run_check(health_state, cvds=[], ports=[]) + health_state[1]["deadline"] = time.monotonic() - 1 + with pytest.raises(RuntimeError, match="timed out"): + run_check(health_state) + + +def test_operation_failure(health_state): + health_state[1]["state"] = "failed" + with pytest.raises(RuntimeError, match="failed"): + run_check(health_state) + + +@pytest.mark.parametrize("state", ["off", "running", "transition"]) +def test_runtime_restart_fails_even_when_api_returns(health_state, state): + health_state[1]["state"] = state + Path(health_state[1]["runtime_id_path"]).write_text("runtime-2") + with pytest.raises(RuntimeError, match="runtime restarted"): + run_check(health_state) + + +def test_listener_state_and_ipv6(): + tcp = "header\n0: 0100007F:1E01 00000000:0000 0A\n1: 0100007F:1C84 00000000:0000 01\n" + tcp6 = "header\n0: 00000000000000000000000000000000:45B1 00000000:0000 0A\n" + with patch.object(Path, "read_text", side_effect=[tcp, tcp6]): + assert listening_ports() == {7681, 17841} + + +def test_warm_exporter_before_first_lease(tmp_path): + from .health import initialize + + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("runtime-1") + state_path = tmp_path / "health.json" + initialize(str(state_path), str(runtime_id), "http://127.0.0.1:2081") + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", return_value=io.BytesIO()): + check(str(state_path)) + runtime_id.write_text("runtime-2") + with pytest.raises(RuntimeError, match="runtime restarted"): + check(str(state_path)) From 0a59502d894a0ff60a9d5ae021d5abf1f10f6465 Mon Sep 17 00:00:00 2001 From: Benny Zlotnik Date: Sun, 6 Sep 2026 08:49:11 +0300 Subject: [PATCH 2/3] feat: add cuttlefish provisioner for dynamic exporters Provision private Cuttlefish runtimes with image preparation, resource budgets, validated ports, and managed-driver health checks. Reconcile ingress isolation before creating Pods to protect lease-controlled APIs. Require a dedicated workload service account, privileged admission, and crosvm userspace VSOCK with netsim. Document SCCs, storage access modes, reproducible image configuration, and lease-aware failure recovery. Signed-off-by: Benny Zlotnik --- .../cmd/exporter-set-controller/main.go | 8 +- .../deploy/operator/config/rbac/role.yaml | 11 + .../controller/jumpstarter/exporterset.go | 5 + .../jumpstarter/exporterset_test.go | 10 + .../jumpstarter/jumpstarter_controller.go | 1 + .../controller/lease_controller_test.go | 2 +- .../exporterset/networkpolicy_test.go | 94 ++ .../provisioners/cuttlefish/README.md | 197 +++++ .../provisioners/cuttlefish/cuttlefish.go | 817 ++++++++++++++++++ .../cuttlefish/cuttlefish_test.go | 434 ++++++++++ controller/internal/exporterset/reconciler.go | 43 +- .../jumpstarter-driver-cuttlefish/README.md | 20 +- 12 files changed, 1623 insertions(+), 19 deletions(-) create mode 100644 controller/internal/exporterset/networkpolicy_test.go create mode 100644 controller/internal/exporterset/provisioners/cuttlefish/README.md create mode 100644 controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go create mode 100644 controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go diff --git a/controller/cmd/exporter-set-controller/main.go b/controller/cmd/exporter-set-controller/main.go index 93eb6a9ef..6bcf56d69 100644 --- a/controller/cmd/exporter-set-controller/main.go +++ b/controller/cmd/exporter-set-controller/main.go @@ -36,6 +36,7 @@ import ( jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset" + "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish" "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/qemu" ) @@ -159,9 +160,14 @@ func main() { // Add new provisioners here as they are implemented. func selectProvisioner(name string) (exporterset.Provisioner, error) { switch name { + case cuttlefish.ProvisionerName: + return cuttlefish.New(version), nil case qemu.ProvisionerName: return qemu.New(version), nil default: - return nil, fmt.Errorf("unknown provisioner %q; supported: %s", name, qemu.ProvisionerName) + return nil, fmt.Errorf( + "unknown provisioner %q; supported: %s, %s", + name, qemu.ProvisionerName, cuttlefish.ProvisionerName, + ) } } diff --git a/controller/deploy/operator/config/rbac/role.yaml b/controller/deploy/operator/config/rbac/role.yaml index 6bc4f8ec7..e6a4cec31 100644 --- a/controller/deploy/operator/config/rbac/role.yaml +++ b/controller/deploy/operator/config/rbac/role.yaml @@ -171,6 +171,17 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - patch + - update + - watch - apiGroups: - operator.jumpstarter.dev resources: diff --git a/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go b/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go index 46a842cc3..4e2501663 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go @@ -517,6 +517,11 @@ func exporterSetPolicyRules() []rbacv1.PolicyRule { Resources: []string{"leases"}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, + }, { APIGroups: []string{""}, Resources: []string{"pods"}, diff --git a/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go b/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go index e58351745..e3dc39d4d 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go @@ -120,6 +120,16 @@ var _ = Describe("exporterSetPolicyRules", func() { Expect(groups).To(HaveKey("coordination.k8s.io")) }) + It("should reconcile runtime network isolation policies", func() { + for _, rule := range rules { + if containsString(rule.APIGroups, "networking.k8s.io") && containsString(rule.Resources, "networkpolicies") { + Expect(rule.Verbs).To(ContainElements("get", "list", "watch", "create", "update", "patch")) + return + } + } + Fail("no rule found for runtime network policies") + }) + It("should grant read-only access on exportersets (no create/update/delete)", func() { for _, rule := range rules { if containsString(rule.APIGroups, "virtualtarget.jumpstarter.dev") && diff --git a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 1a15ddf47..3e7706e9d 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -90,6 +90,7 @@ type JumpstarterReconciler struct { // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete // Networking resources +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete diff --git a/controller/internal/controller/lease_controller_test.go b/controller/internal/controller/lease_controller_test.go index 26e3d6200..e207c61e5 100755 --- a/controller/internal/controller/lease_controller_test.go +++ b/controller/internal/controller/lease_controller_test.go @@ -1492,7 +1492,7 @@ var _ = Describe("Scheduled Leases", func() { When("creating lease with BeginTime + Duration (scheduled lease)", func() { It("should wait until BeginTime before acquiring exporter", func() { lease := leaseDutA2Sec.DeepCopy() - futureTime := metav1.NewTime(time.Now().Truncate(time.Second).Add(1 * time.Second)) + futureTime := metav1.NewTime(time.Now().Add(2 * time.Second).Truncate(time.Second)) lease.Spec.BeginTime = &futureTime lease.Spec.Duration = &metav1.Duration{Duration: 1 * time.Second} lease.Spec.EndTime = nil diff --git a/controller/internal/exporterset/networkpolicy_test.go b/controller/internal/exporterset/networkpolicy_test.go new file mode 100644 index 000000000..102dbf2ca --- /dev/null +++ b/controller/internal/exporterset/networkpolicy_test.go @@ -0,0 +1,94 @@ +package exporterset + +import ( + "context" + "errors" + "strings" + "testing" + + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestCuttlefishNetworkPolicyReconciliation(t *testing.T) { + scheme := newScheme(t) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + es := makeExporterSet() + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &ExporterSetReconciler{Client: c, Scheme: scheme, Provisioner: cuttlefish.New("dev")} + ctx := context.Background() + if err := r.syncNetworkPolicy(ctx, es); err != nil { + t.Fatal(err) + } + policy := &networkingv1.NetworkPolicy{} + key := client.ObjectKey{Namespace: es.Namespace, Name: "cuttlefish-" + string(es.UID)} + if err := c.Get(ctx, key, policy); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(policy, es) || len(policy.Spec.Ingress) != 0 { + t.Fatalf("unowned or permissive policy: %#v", policy) + } + policy.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{{}} + if err := c.Update(ctx, policy); err != nil { + t.Fatal(err) + } + if err := r.syncNetworkPolicy(ctx, es); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, key, policy); err != nil { + t.Fatal(err) + } + if len(policy.Spec.Ingress) != 0 { + t.Fatal("policy drift not corrected") + } + if err := c.Delete(ctx, policy); err != nil { + t.Fatal(err) + } + if err := r.syncNetworkPolicy(ctx, es); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, key, policy); err != nil { + t.Fatal("deleted policy not recreated", err) + } +} + +func TestPolicyFailurePreventsWorkloadCreation(t *testing.T) { + scheme := newScheme(t) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + es := makeExporterSet() + vtc := &virtualtargetv1alpha1.VirtualTargetClass{ + ObjectMeta: metav1.ObjectMeta{Name: es.Spec.VirtualTargetClassName, Namespace: es.Namespace}, + Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Provisioner: cuttlefish.ProvisionerName}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(es, vtc).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*networkingv1.NetworkPolicy); ok { + return errors.New("network policy denied") + } + return c.Create(ctx, obj, opts...) + }, + }).Build() + r := &ExporterSetReconciler{Client: c, Scheme: scheme, Provisioner: cuttlefish.New("dev")} + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(es)}) + if err == nil || !strings.Contains(err.Error(), "network policy denied") { + t.Fatalf("expected policy error: %v", err) + } + var pods corev1.PodList + if err := c.List(context.Background(), &pods); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatal("created workload without network isolation") + } +} diff --git a/controller/internal/exporterset/provisioners/cuttlefish/README.md b/controller/internal/exporterset/provisioners/cuttlefish/README.md new file mode 100644 index 000000000..ba7937c00 --- /dev/null +++ b/controller/internal/exporterset/provisioners/cuttlefish/README.md @@ -0,0 +1,197 @@ +# Cuttlefish ExporterSets + +Each exporter owns one CVD. The managed backend uses Host Orchestrator over +HTTP inside the Pod, crosvm with private userspace VSOCK, and netsim Bluetooth. +The standalone Python driver continues to support externally managed HTTP hosts. +An exec backend is a separate follow-up. + +## Workload admission and networking + +Create a dedicated service account in the ExporterSet namespace and set +`parameters.service_account_name` to its name. `default` is rejected. The Pod +does not mount a Kubernetes API token. `runtime_privileged: true` is required; +`false` is rejected until device permissions and capabilities are supported. + +On Kubernetes, create the namespace and workload account. If Pod Security +Admission is enabled, the namespace must permit privileged workloads; the +`baseline` and `restricted` profiles reject this Pod. For a dedicated test +namespace: + +```sh +kubectl create namespace cuttlefish-lab +kubectl label namespace cuttlefish-lab pod-security.kubernetes.io/enforce=privileged +kubectl -n cuttlefish-lab create serviceaccount cuttlefish-runtime +``` + +This namespace setting permits privileged workloads for every account in that +namespace, so restrict who can create workloads there. Other admission policies +must also allow the Pod's privileged containers, hostPath devices and container +UIDs. The workload account needs no Kubernetes API permissions. Nodes must expose +the required KVM and networking devices; VM-based nodes need nested virtualization. +The cluster must support native sidecar containers: the runtime init containers +use `restartPolicy: Always`. +See [Kubernetes Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). + +On OpenShift, a cluster administrator must grant that workload account access +to an SCC permitting privileged containers, hostPath devices, and the UIDs used +by **all** containers, including image copy/fetch and permission init containers. +For initial testing, a scoped grant to the built-in privileged SCC is: + +```sh +oc -n cuttlefish-lab create serviceaccount cuttlefish-runtime +oc adm policy add-scc-to-user privileged -z cuttlefish-runtime -n cuttlefish-lab +``` + +This grant is for the workload account, not the ExporterSet controller. Merely +setting `runtime_privileged` cannot grant SCC admission. Check the admitted Pod's +`openshift.io/scc` annotation, completion of init containers, and an actual guest +boot. A server-side dry run only establishes admission, not device access. +See [OpenShift SCC documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/authentication_and_authorization/managing-pod-security-policies). + +The controller creates and reconciles an ExporterSet-owned NetworkPolicy before +creating workloads. It denies Pod ingress, including Host Orchestrator, nginx, +ADB and simulator listeners, while leaving outbound exporter connections and +same-Pod loopback traffic available. Deploy with a CNI that enforces NetworkPolicy. +Other policies must not grant ingress to these Pods: Kubernetes allow rules are +additive. Verify denial from a second Pod, using the actual runtime image; +loopback addresses in driver configuration do not change the server's listeners. +NetworkPolicy does not isolate privileged containers from their node. + +Controllers deployed outside the operator also need `get,list,watch,create,update,patch` +on `networking.k8s.io/networkpolicies`. The operator supplies these permissions. +Existing Pods must be drained and replaced to receive the isolation label, +service account, runtime marker and managed driver configuration. + +## Configuration and resource allocation + +The provisioner requires exactly one Cuttlefish driver and one +`env_config.instances` entry. It fixes the managed endpoint to +`http://127.0.0.1:2081` and `instance_num` to 1. A different +`host_orchestrator_port` is rejected because the upstream image fixes its service +and nginx upstream configuration to 2081. + +Guest defaults are 4 CPUs and 8192 MiB. Runtime memory requests default to the +**effective** guest memory plus `runtime_memory_overhead_mb` (2048 MiB by default). +Explicit requests and limits must cover that budget. Increase the overhead for +larger simulator workloads. Guest values in driver `env_config` take precedence +over `vm_cpus` and `vm_memory_mb` when calculating the budget. CPU requests default +to guest CPUs, or to an explicit CPU limit; CPU overcommit remains configurable. + +Relay ports must be distinct integers in 1..65535 and must not collide with the +managed runtime's service ports. Defaults are 17681 (netsim) and 17300 (HCI). +The relay supervisor exits if either listener process dies. + +`create_cvd()` in managed mode accepts only the exact configured `env_config`, +checks the entire Host Orchestrator inventory, and serializes creation with other +lifecycle operations. Destroy the existing CVD before creating another. This +prevents alternate API payloads or concurrent calls from exceeding the configured +instance count and memory budget. Arbitrary template code and driver imports are +administrator-controlled; they are not a security boundary against a malicious +cluster administrator. + +## VSOCK and Bluetooth compatibility + +Managed configuration sets: + +```yaml +env_config: + netsim_bt: true + instances: + - vm: + crosvm: + vhost_user_vsock: "true" +``` + +The string value is required by the upstream configuration schema. The runtime +and guest must support that backend. The provisioner does not mount +`/dev/vhost-vsock`; each Pod has private runtime files and Unix sockets. A +privileged runtime still has broad node access, so this is not containment of a +compromised runtime. QEMU, gem5, disabled userspace VSOCK, and standalone RootCanal +(`netsim_bt: false`) are rejected for this managed backend. The referenced +upstream standalone RootCanal proxy does not propagate the userspace VSOCK flag. + +Before approving a runtime/build pair, boot two Pods on the same node using the +default identical guest CID, verify their generated configuration and private +`vhost.socket`/`vm.vsock` paths, and exercise netsim plus Bluetooth peer traffic +in both leases. Stop one guest and verify the other remains usable. Source/unit +tests cannot establish compatibility of a mutable image tag or guest build. + +## Image PVCs and reproducibility + +A prewarmed image PVC is mounted read-only, then copied into a private writable +`emptyDir`. It remains a Pod volume after the init container exits. Read-only +mounting does **not** change the PVC's access mode or remove attachment constraints. + +For a pool spanning nodes, use storage supporting ReadOnlyMany or ReadWriteMany +with the required topology. For ReadWriteOnce, keep readers on one compatible +node; ReadWriteOncePod permits only one Pod. Alternatively fetch images per Pod. +See [Kubernetes access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). + +Use an immutable runtime manifest digest, relay digest and Android build ID for +repeatable replacements. This example is a template: substitute verified values +from a tested pair; the placeholders are not a published compatibility claim. +Pin the exporter image built with this provisioner/driver change too. + +```yaml +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: VirtualTargetClass +metadata: + name: cuttlefish + namespace: cuttlefish-lab +spec: + provisioner: cuttlefish.jumpstarter.dev + parameters: + service_account_name: cuttlefish-runtime + runtime_privileged: true + fetch_images: true + default_build: "/aosp_cf_x86_64_auto-userdebug" + relay_image: "docker.io/alpine/socat@sha256:" + vm_cpus: 4 + vm_memory_mb: 8192 + runtime_memory_overhead_mb: 2048 + images: + runtime: + image: "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orchestration/cuttlefish-orchestration@sha256:" + exporter: + image: "quay.io/jumpstarter-dev/jumpstarter@sha256:" + scheduling: + resources: + requests: + cpu: "4" + memory: 10Gi + limits: + memory: 10Gi +``` + +Record both the resolved runtime image digest and the guest `fetcher_config.json` +with validation results. A prewarmed PVC needs the same build provenance. + +## Failure and recovery behavior + +The exporter liveness probe reads state written atomically by the managed driver. +It always checks Host Orchestrator availability and a per-start runtime ID. +When the guest is expected to run, it also checks the CVD inventory/status and +simulator/relay TCP listeners in the shared network namespace. It inspects +listeners instead of opening HCI connections that could disturb Bluetooth peers. +A listening socket alone does not prove simulator protocol correctness. + +Intentional stop, destroy and reset leave the Pod healthy without a guest. +Create/start/restart/powerwash receive bounded transition time; failed or expired +operations fail health checks. A runtime sidecar restart invalidates the exporter +even if its HTTP API comes back: it must not silently resume a lease after losing +runtime state. Six failed checks, ten seconds apart, terminate the exporter. + +With `ExitAndReplace`, the failed exporter remains associated with an active +lease. Release that lease to let the controller delete and replace the Pod; +reacquire a lease for a fresh device. Automatic replacement during an active +lease is not attempted. Use `ExitAndReplace` for managed Cuttlefish pools. + +Validate recovery on disposable leased Pods by terminating the VMM, netsim, each +relay, and then the runtime sidecar in separate trials. A component may recover +within the probe failure window; verify guest and simulator functionality after +recovery. For unrecovered failures and runtime sidecar restarts, confirm liveness +failure, client disconnect, retention while leased, and replacement after release. Also +exercise intentional power off/on and destroy/create, which must remain healthy. +Actual CNI enforcement and colocated VSOCK/BT trials are required integration +checks beyond the local regression tests. OpenShift deployments additionally +require SCC admission and guest boot validation. diff --git a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go new file mode 100644 index 000000000..acea04d65 --- /dev/null +++ b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go @@ -0,0 +1,817 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cuttlefish implements the cuttlefish.jumpstarter.dev provisioner. +// Each exporter Pod owns one Cuttlefish runtime and one CVD. +package cuttlefish + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "math" + "slices" + "strings" + + jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + ProvisionerName = "cuttlefish.jumpstarter.dev" + + DefaultExporterImage = "quay.io/jumpstarter-dev/jumpstarter:latest" + DefaultRuntimeImage = "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orchestration/cuttlefish-orchestration:stable" + DefaultRelayImage = "docker.io/alpine/socat:latest" + + exporterConfigPath = "/etc/jumpstarter/exporters/config.yaml" + exporterNonRootUID int64 = 65532 + + cuttlefishDriverType = "jumpstarter_driver_cuttlefish.driver.Cuttlefish" + netsimDriverType = "jumpstarter_driver_netsim.driver.Netsim" + btPeerDriverType = "jumpstarter_driver_bt_peer.driver.BtPeer" + + // The orchestration image reserves 2080 for nginx when running in a Pod; + // Host Orchestrator consequently listens on 2081. Keep this configurable. + hostOrchestratorPort = 2081 + netsimRelayPort = 17681 + hciRelayPort = 17300 + defaultGPUMode = "guest_swiftshader" + defaultVMCPUs = 4 + defaultVMMemoryMB = 8192 + isolationLabel = "cuttlefish.jumpstarter.dev/exporter-set" + healthStatePath = "/tmp/jumpstarter-cuttlefish-health.json" + runtimeIDPath = "/run/cuttlefish-runtime/runtime-id" + + fetchPath = "/home/vsoc-01/fetch" + cvdStatePath = "/var/tmp/cvd" + androidTmpPath = "/tmp/android" +) + +type Provisioner struct { + Version string +} + +type storageConfig struct { + imageClaim string + fetchImages bool + imageSize, stateSize, tmpSize, budget resource.Quantity + build string +} + +type runtimeConfig struct { + relayImage string + netsimPort, hciPort int + privileged bool + serviceAccount string +} + +func New(version string) *Provisioner { + return &Provisioner{Version: version} +} + +func (p *Provisioner) Name() string { + return ProvisionerName +} + +func (p *Provisioner) resolveImage(image string) string { + if p.Version == "" || p.Version == "dev" || strings.Contains(p.Version, "-g") { + return image + } + version := strings.TrimPrefix(p.Version, "v") + if base, ok := strings.CutSuffix(image, ":latest"); ok { + return base + ":" + version + } + return image +} + +func (p *Provisioner) resolveImageSpec(spec *virtualtargetv1alpha1.ImageSpec, defaultImage string) (string, corev1.PullPolicy) { + image := p.resolveImage(defaultImage) + pullPolicy := corev1.PullIfNotPresent + if spec != nil { + if spec.Image != "" { + image = spec.Image + } + if spec.ImagePullPolicy != "" { + pullPolicy = spec.ImagePullPolicy + } + } + return image, pullPolicy +} + +func resolveStorageConfig(parameters map[string]interface{}) (storageConfig, error) { + config := storageConfig{} + config.imageClaim, _ = parameters["image_volume_claim"].(string) + config.fetchImages, _ = parameters["fetch_images"].(bool) + if config.imageClaim != "" && config.fetchImages { + return config, fmt.Errorf("cuttlefish requires either image_volume_claim or fetch_images, not both") + } + if config.imageClaim == "" && !config.fetchImages { + return config, fmt.Errorf("cuttlefish requires image_volume_claim or fetch_images=true") + } + if readOnly, _ := parameters["image_volume_read_only"].(bool); readOnly && config.imageClaim == "" { + return config, fmt.Errorf("image_volume_read_only requires image_volume_claim") + } + + var err error + config.imageSize, config.stateSize, config.tmpSize, err = storageSizes(parameters) + if err != nil { + return config, err + } + config.budget = config.imageSize.DeepCopy() + config.budget.Add(config.stateSize) + config.budget.Add(config.tmpSize) + config.budget.Add(resource.MustParse("1Gi")) // Container layers and logs need space beyond the volume budgets. + if config.fetchImages { + config.build = resolveDefaultBuild(parameters) + } + return config, nil +} + +func resolveRuntimeConfig(parameters map[string]interface{}) (runtimeConfig, error) { + config := runtimeConfig{relayImage: DefaultRelayImage} + if value, ok := parameters["relay_image"].(string); ok && value != "" { + config.relayImage = value + } + + var err error + config.netsimPort, config.hciPort, err = relayPorts(parameters) + if err != nil { + return config, err + } + configured := false + config.privileged, configured = parameterBool(parameters, "runtime_privileged") + if !configured || !config.privileged { + return config, fmt.Errorf("cuttlefish requires runtime_privileged=true; unprivileged device access is not supported") + } + + config.serviceAccount, _ = parameters["service_account_name"].(string) + if config.serviceAccount == "" || config.serviceAccount == "default" || len(validation.IsDNS1123Subdomain(config.serviceAccount)) != 0 { + return config, fmt.Errorf("service_account_name must name a dedicated workload service account") + } + return config, nil +} + +func (p *Provisioner) RenderPod( + ctx context.Context, + exporterSet *virtualtargetv1alpha1.ExporterSet, + vtc *virtualtargetv1alpha1.VirtualTargetClass, + mergedParameters map[string]interface{}, + images *virtualtargetv1alpha1.ImageOverrides, + exporter *jumpstarterdevv1alpha1.Exporter, +) (*corev1.Pod, error) { + _ = ctx + if exporterSet.Spec.RecycleStrategy == virtualtargetv1alpha1.RecycleStrategyInPlaceReuse { + return nil, fmt.Errorf("managed Cuttlefish requires ExitAndReplace recycling") + } + + var exporterSpec, runtimeSpec *virtualtargetv1alpha1.ImageSpec + if images != nil { + exporterSpec = images.Exporter + runtimeSpec = images.Runtime + } + exporterImage, exporterPullPolicy := p.resolveImageSpec(exporterSpec, DefaultExporterImage) + runtimeImage, runtimePullPolicy := p.resolveImageSpec(runtimeSpec, DefaultRuntimeImage) + storage, err := resolveStorageConfig(mergedParameters) + if err != nil { + return nil, err + } + runtime, err := resolveRuntimeConfig(mergedParameters) + if err != nil { + return nil, err + } + drivers, err := p.EnrichExporterExport(exporterSet.Spec.Template.Spec.Drivers, mergedParameters) + if err != nil { + return nil, err + } + runtimeResources := corev1.ResourceRequirements{} + if vtc.Spec.Scheduling != nil && vtc.Spec.Scheduling.Resources != nil { + runtimeResources = *vtc.Spec.Scheduling.Resources.DeepCopy() + } + if err := reserveRuntimeResources(&runtimeResources, drivers, mergedParameters); err != nil { + return nil, err + } + + podMeta := metav1.ObjectMeta{ + Namespace: exporterSet.Namespace, + Labels: maps.Clone(exporterSet.Spec.Template.Metadata.Labels), + Annotations: maps.Clone(exporterSet.Spec.Template.Metadata.Annotations), + } + if exporter != nil { + podMeta.Name = exporter.Name + } else { + podMeta.GenerateName = fmt.Sprintf("%s-", exporterSet.Name) + } + + if podMeta.Labels == nil { + podMeta.Labels = map[string]string{} + } + podMeta.Labels[isolationLabel] = string(exporterSet.UID) + + runtimeRestart := corev1.ContainerRestartPolicyAlways + runAsRoot := int64(0) + runAsExporter := exporterNonRootUID + runAsNonRoot := true + + volumeMounts := []corev1.VolumeMount{ + {Name: "cvd-images", MountPath: fetchPath, ReadOnly: false}, + {Name: "cvd-state", MountPath: cvdStatePath}, + {Name: "android-tmp", MountPath: androidTmpPath}, + } + deviceMounts := []corev1.VolumeMount{ + {Name: "kvm", MountPath: "/dev/kvm"}, + {Name: "vhost-net", MountPath: "/dev/vhost-net"}, + {Name: "tun", MountPath: "/dev/net/tun"}, + } + + exporterContainer := corev1.Container{ + Name: "exporter", + VolumeMounts: []corev1.VolumeMount{{Name: "cvd-state", MountPath: "/run/cuttlefish-runtime", ReadOnly: true}}, + Image: exporterImage, + ImagePullPolicy: exporterPullPolicy, + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", "--run-exporter", + healthStatePath, runtimeIDPath, "http://127.0.0.1:2081", exporterConfigPath}, + Env: []corev1.EnvVar{{ + Name: "HOME", + Value: "/tmp", + }}, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsExporter, + RunAsNonRoot: &runAsNonRoot, + }, + } + if exporter != nil { + exporterContainer.Env = append(exporterContainer.Env, corev1.EnvVar{ + Name: "JUMPSTARTER_EXEC_LOG_FIELDS", + Value: fmt.Sprintf("component=exporter,exporter=%s,namespace=%s", exporter.Name, exporter.Namespace), + }) + } + + imageVolume := corev1.Volume{Name: "cvd-images", VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &storage.imageSize}, + }} + permissionCommand := "mkdir -p /var/tmp/cvd /tmp/android && chown -R httpcvd:httpcvd /var/tmp/cvd /tmp/android /home/vsoc-01/fetch" + + initContainers := make([]corev1.Container, 0, 4) + if storage.fetchImages { + initContainers = append(initContainers, corev1.Container{ + Name: "fetch-images", + Image: runtimeImage, + ImagePullPolicy: runtimePullPolicy, + Command: []string{"cvd", "fetch", "--default_build=" + storage.build, "--target_directory=" + fetchPath}, + VolumeMounts: []corev1.VolumeMount{{Name: "cvd-images", MountPath: fetchPath, ReadOnly: false}}, + }) + } + initContainers = append(initContainers, + corev1.Container{ + Name: "fix-cuttlefish-permissions", + Image: runtimeImage, + ImagePullPolicy: runtimePullPolicy, + Command: []string{ + "bash", "-c", permissionCommand, + }, + VolumeMounts: volumeMounts, + }, + corev1.Container{ + Name: "cuttlefish", + Image: runtimeImage, + ImagePullPolicy: runtimePullPolicy, + RestartPolicy: &runtimeRestart, + Command: []string{"bash", "-ec", `cat /proc/sys/kernel/random/uuid > /var/tmp/cvd/runtime-id +chmod 644 /var/tmp/cvd/runtime-id +exec /root/run_services.sh`}, + Resources: runtimeResources, + SecurityContext: &corev1.SecurityContext{Privileged: boolPtr(runtime.privileged), RunAsUser: &runAsRoot}, + VolumeMounts: slices.Concat(volumeMounts, deviceMounts), + }, + corev1.Container{ + Name: "cuttlefish-relay", + Image: runtime.relayImage, + ImagePullPolicy: corev1.PullIfNotPresent, + RestartPolicy: &runtimeRestart, + Command: []string{ + "sh", "-c", + fmt.Sprintf( + `socat TCP-LISTEN:%d,bind=127.0.0.1,fork,reuseaddr TCP:127.0.0.1:7681 & +first=$! +socat TCP-LISTEN:%d,bind=127.0.0.1,fork,reuseaddr TCP:127.0.0.1:7300 & +second=$! +trap 'kill $first $second 2>/dev/null || true' EXIT +while kill -0 $first && kill -0 $second; do sleep 1; done +exit 1`, + runtime.netsimPort, runtime.hciPort, + ), + }, + }, + ) + + pod := &corev1.Pod{ + ObjectMeta: podMeta, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: runtime.serviceAccount, + AutomountServiceAccountToken: boolPtr(false), + InitContainers: initContainers, + Containers: []corev1.Container{exporterContainer}, + Volumes: []corev1.Volume{ + imageVolume, + {Name: "cvd-state", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &storage.stateSize}}}, + {Name: "android-tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &storage.tmpSize}}}, + deviceVolume("kvm", "/dev/kvm"), + deviceVolume("vhost-net", "/dev/vhost-net"), + deviceVolume("tun", "/dev/net/tun"), + }, + }, + } + + if vtc.Spec.Scheduling != nil { + if vtc.Spec.Scheduling.NodeSelector != nil { + pod.Spec.NodeSelector = maps.Clone(vtc.Spec.Scheduling.NodeSelector) + } + if vtc.Spec.Scheduling.Tolerations != nil { + pod.Spec.Tolerations = append([]corev1.Toleration(nil), vtc.Spec.Scheduling.Tolerations...) + } + } + + if storage.imageClaim != "" { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{Name: "image-source", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: storage.imageClaim, ReadOnly: true}, + }}) + copyImages := corev1.Container{ + Name: "copy-images", Image: runtimeImage, ImagePullPolicy: runtimePullPolicy, + Command: []string{"bash", "-ec", "cp -a --reflink=auto /image-source/. /home/vsoc-01/fetch/"}, + VolumeMounts: []corev1.VolumeMount{ + {Name: "image-source", MountPath: "/image-source", ReadOnly: true}, + {Name: "cvd-images", MountPath: fetchPath}, + }, + } + pod.Spec.InitContainers = append([]corev1.Container{copyImages}, pod.Spec.InitContainers...) + } + for i := range pod.Spec.InitContainers { + container := &pod.Spec.InitContainers[i] + if container.Name == "cuttlefish" || container.Name == "fetch-images" || container.Name == "copy-images" { + if err := reserveStorage(&container.Resources, storage.budget); err != nil { + return nil, err + } + } + } + // Run in the exporter image so the check uses the same network namespace and Python runtime as jmp. + healthURL := fmt.Sprintf("http://127.0.0.1:%d/_debug/statusz", parameterInt(mergedParameters, "host_orchestrator_port", hostOrchestratorPort)) + healthCheck := "import urllib.request; urllib.request.urlopen(" + fmt.Sprintf("%q", healthURL) + ", timeout=3).close()" + pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ + Name: "wait-for-cuttlefish", Image: exporterImage, ImagePullPolicy: exporterPullPolicy, + SecurityContext: exporterContainer.SecurityContext.DeepCopy(), + Command: []string{"python3", "-c", "import time, urllib.request\nfor attempt in range(60):\n try:\n " + healthCheck + "\n break\n except Exception:\n time.sleep(5)\nelse:\n raise SystemExit('Host Orchestrator did not become ready')"}, + }) + // With restartPolicy Never, a failed liveness check ends the exporter and lets ExitAndReplace recycle the Pod. + pod.Spec.Containers[0].LivenessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", healthStatePath}}}, + PeriodSeconds: 10, TimeoutSeconds: 10, FailureThreshold: 6, + } + + return pod, nil +} + +func (p *Provisioner) EnrichExporterExport( + drivers []virtualtargetv1alpha1.DriverConfig, + mergedParameters map[string]interface{}, +) ([]virtualtargetv1alpha1.DriverConfig, error) { + netsimPort, hciPort, err := relayPorts(mergedParameters) + if err != nil { + return nil, err + } + count := 0 + for _, driver := range drivers { + if driver.Type == cuttlefishDriverType { + count++ + } + } + if count != 1 { + return nil, fmt.Errorf("cuttlefish requires exactly one Cuttlefish driver per Pod, got %d", count) + } + result := make([]virtualtargetv1alpha1.DriverConfig, 0, len(drivers)) + for _, driver := range drivers { + var err error + switch driver.Type { + case cuttlefishDriverType: + driver, err = enrichCuttlefishDriver(driver, mergedParameters) + case netsimDriverType: + driver, err = enrichDriverConfig(driver, map[string]interface{}{ + "host": "127.0.0.1", + "port": netsimPort, + }, "netsim") + case btPeerDriverType: + driver, err = enrichDriverConfig(driver, map[string]interface{}{ + "transport": fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort), + }, "bt_peer") + } + if err != nil { + return nil, err + } + result = append(result, driver) + } + return result, nil +} + +func enrichCuttlefishDriver(driver virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) (virtualtargetv1alpha1.DriverConfig, error) { + for _, item := range []struct { + key string + fallback int + }{{"vm_cpus", defaultVMCPUs}, {"vm_memory_mb", defaultVMMemoryMB}} { + if _, err := positiveInt(parameters, item.key, item.fallback); err != nil { + return driver, err + } + } + config, err := decodeConfig(driver, "Cuttlefish") + if err != nil { + return driver, err + } + + config["managed"] = true + config["health_state_path"] = healthStatePath + config["runtime_id_path"] = runtimeIDPath + config["health_ports"] = []int{7681, 7300, parameterInt(parameters, "netsim_relay_port", netsimRelayPort), parameterInt(parameters, "hci_relay_port", hciRelayPort)} + setDefault(config, "scheme", "http") + setDefault(config, "host", "127.0.0.1") + setDefault(config, "port", parameterInt(parameters, "host_orchestrator_port", hostOrchestratorPort)) + setDefault(config, "group", "cvd") + setDefault(config, "name", "1") + setDefault(config, "instance_num", 1) + setDefault(config, "boot_timeout", 300) + if err := validateManagedEndpoint(config); err != nil { + return driver, err + } + + envConfig, err := configObject(config, "env_config") + if err != nil { + return driver, err + } + common, err := configObject(envConfig, "common") + if err != nil { + return driver, err + } + setDefault(common, "host_package", fetchPath) + envConfig["common"] = common + instances, ok := envConfig["instances"].([]interface{}) + if raw, exists := envConfig["instances"]; exists && (!ok || len(instances) != 1) { + return driver, fmt.Errorf("env_config.instances must contain exactly one instance, got %v", raw) + } + if len(instances) == 0 { + instances = []interface{}{map[string]interface{}{}} + } + instance, ok := instances[0].(map[string]interface{}) + if !ok || instance == nil { + return driver, fmt.Errorf("env_config.instances[0] must be an object") + } + disk, err := configObject(instance, "disk") + if err != nil { + return driver, err + } + setDefault(disk, "default_build", fetchPath) + instance["disk"] = disk + graphics, err := configObject(instance, "graphics") + if err != nil { + return driver, err + } + gpuMode := defaultGPUMode + if configuredGPU, ok := parameters["gpu_mode"].(string); ok && configuredGPU != "" { + gpuMode = configuredGPU + } + setDefault(graphics, "gpu_mode", gpuMode) + instance["graphics"] = graphics + vm, err := configObject(instance, "vm") + if err != nil { + return driver, err + } + if _, exists := vm["qemu"]; exists { + return driver, fmt.Errorf("managed Cuttlefish requires crosvm with private userspace VSOCK") + } + if _, exists := vm["gem5"]; exists { + return driver, fmt.Errorf("managed Cuttlefish requires crosvm with private userspace VSOCK") + } + crosvm, err := configObject(vm, "crosvm") + if err != nil { + return driver, err + } + if value, exists := crosvm["vhost_user_vsock"]; exists && value != "true" { + return driver, fmt.Errorf("vm.crosvm.vhost_user_vsock must be the string true") + } + crosvm["vhost_user_vsock"] = "true" + vm["crosvm"] = crosvm + if value, exists := envConfig["netsim_bt"]; exists && value != true { + return driver, fmt.Errorf("managed Cuttlefish requires netsim_bt=true; standalone RootCanal is not supported") + } + envConfig["netsim_bt"] = true + setDefault(vm, "cpus", parameterInt(parameters, "vm_cpus", defaultVMCPUs)) + setDefault(vm, "memory_mb", parameterInt(parameters, "vm_memory_mb", defaultVMMemoryMB)) + if _, err := positiveInt(vm, "cpus", defaultVMCPUs); err != nil { + return driver, err + } + if _, err := positiveInt(vm, "memory_mb", defaultVMMemoryMB); err != nil { + return driver, err + } + instance["vm"] = vm + instances[0] = instance + envConfig["instances"] = instances + config["env_config"] = envConfig + + return encodeConfig(driver, config) +} + +func validateManagedEndpoint(config map[string]interface{}) error { + for key, required := range map[string]interface{}{"scheme": "http", "host": "127.0.0.1"} { + if config[key] != required { + return fmt.Errorf("managed Cuttlefish requires %s=%v", key, required) + } + } + if port, err := positiveInt(config, "port", hostOrchestratorPort); err != nil || port != hostOrchestratorPort { + return fmt.Errorf("managed Cuttlefish requires port=%d", hostOrchestratorPort) + } + if instanceNum, err := positiveInt(config, "instance_num", 1); err != nil || instanceNum != 1 { + return fmt.Errorf("managed Cuttlefish requires instance_num=1") + } + return nil +} + +func configObject(parent map[string]interface{}, key string) (map[string]interface{}, error) { + raw, exists := parent[key] + if !exists { + return map[string]interface{}{}, nil + } + value, ok := raw.(map[string]interface{}) + if !ok || value == nil { + return nil, fmt.Errorf("%s must be an object", key) + } + return value, nil +} + +func enrichDriverConfig(driver virtualtargetv1alpha1.DriverConfig, defaults map[string]interface{}, name string) (virtualtargetv1alpha1.DriverConfig, error) { + config, err := decodeConfig(driver, name) + if err != nil { + return driver, err + } + for key, value := range defaults { + setDefault(config, key, value) + } + return encodeConfig(driver, config) +} + +func decodeConfig(driver virtualtargetv1alpha1.DriverConfig, name string) (map[string]interface{}, error) { + config := map[string]interface{}{} + if driver.Config != nil && driver.Config.Raw != nil { + if err := json.Unmarshal(driver.Config.Raw, &config); err != nil { + return nil, fmt.Errorf("unmarshal %s driver config: %w", name, err) + } + } + if config == nil { + return nil, fmt.Errorf("%s config must be an object", name) + } + return config, nil +} + +func encodeConfig(driver virtualtargetv1alpha1.DriverConfig, config map[string]interface{}) (virtualtargetv1alpha1.DriverConfig, error) { + raw, err := json.Marshal(config) + if err != nil { + return driver, fmt.Errorf("marshal driver config: %w", err) + } + driver.Config = &apiextensionsv1.JSON{Raw: raw} + return driver, nil +} + +func resolveDefaultBuild(parameters map[string]interface{}) string { + if value, ok := parameters["default_build"].(string); ok && value != "" { + return value + } + return "aosp-android-latest-release/aosp_cf_x86_64_auto-userdebug" +} + +func relayPorts(parameters map[string]interface{}) (int, int, error) { + ports := []int{netsimRelayPort, hciRelayPort} + reserved := map[int]bool{ + 80: true, 443: true, 1080: true, 1443: true, 2080: true, 2081: true, 2443: true, + 7300: true, 7301: true, 7302: true, 7303: true, 7681: true, 15037: true, 19531: true, + } + if port, err := positiveInt(parameters, "host_orchestrator_port", hostOrchestratorPort); err != nil || port != hostOrchestratorPort { + return 0, 0, fmt.Errorf("host_orchestrator_port must be %d for the orchestration image", hostOrchestratorPort) + } + for i, key := range []string{"netsim_relay_port", "hci_relay_port"} { + port, err := positiveInt(parameters, key, ports[i]) + if err != nil || port > 65535 || reserved[port] || (port >= 6520 && port <= 6620) || (port >= 15550 && port <= 15560) { + return 0, 0, fmt.Errorf("%s must be an integer port in 1..65535 that does not conflict with a runtime service", key) + } + ports[i] = port + reserved[port] = true + } + return ports[0], ports[1], nil +} + +func positiveInt(values map[string]interface{}, key string, fallback int) (int, error) { + raw, exists := values[key] + if !exists { + return fallback, nil + } + var value float64 + switch v := raw.(type) { + case int: + value = float64(v) + case int32: + value = float64(v) + case int64: + value = float64(v) + case float64: + value = v + default: + return 0, fmt.Errorf("%s must be a positive integer", key) + } + if math.IsNaN(value) || math.IsInf(value, 0) || value < 1 || value > math.MaxInt32 || math.Trunc(value) != value { + return 0, fmt.Errorf("%s must be a positive integer", key) + } + return int(value), nil +} + +func reserveRuntimeResources(resources *corev1.ResourceRequirements, drivers []virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) error { + var vm map[string]interface{} + for _, driver := range drivers { + if driver.Type != cuttlefishDriverType { + continue + } + config, err := decodeConfig(driver, "Cuttlefish") + if err != nil { + return err + } + vm = config["env_config"].(map[string]interface{})["instances"].([]interface{})[0].(map[string]interface{})["vm"].(map[string]interface{}) + } + memory, err := positiveInt(vm, "memory_mb", defaultVMMemoryMB) + if err != nil { + return err + } + cpus, err := positiveInt(vm, "cpus", defaultVMCPUs) + if err != nil { + return err + } + overhead, err := positiveInt(parameters, "runtime_memory_overhead_mb", 2048) + if err != nil { + return err + } + budget := *resource.NewQuantity((int64(memory)+int64(overhead))*1024*1024, resource.BinarySI) + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { + if value, exists := values[corev1.ResourceMemory]; exists && value.Cmp(budget) < 0 { + return fmt.Errorf("runtime memory must be at least %s for guest plus overhead", budget.String()) + } + } + if _, exists := resources.Requests[corev1.ResourceMemory]; !exists { + resources.Requests[corev1.ResourceMemory] = budget + } + if limit, exists := resources.Limits[corev1.ResourceMemory]; exists && resources.Requests.Memory().Cmp(limit) > 0 { + return fmt.Errorf("runtime memory request exceeds limit") + } + if _, exists := resources.Requests[corev1.ResourceCPU]; !exists { + if limit, exists := resources.Limits[corev1.ResourceCPU]; exists { + resources.Requests[corev1.ResourceCPU] = limit.DeepCopy() + } else { + resources.Requests[corev1.ResourceCPU] = *resource.NewQuantity(int64(cpus), resource.DecimalSI) + } + } + + if resources.Requests.Cpu().Sign() <= 0 { + return fmt.Errorf("runtime CPU request must be positive") + } + if limit, exists := resources.Limits[corev1.ResourceCPU]; exists && resources.Requests.Cpu().Cmp(limit) > 0 { + return fmt.Errorf("runtime CPU request exceeds limit") + } + return nil +} + +func (p *Provisioner) RenderNetworkPolicy(es *virtualtargetv1alpha1.ExporterSet) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "cuttlefish-" + string(es.UID), Namespace: es.Namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{isolationLabel: string(es.UID)}}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + }, + } +} + +func parameterInt(parameters map[string]interface{}, key string, fallback int) int { + switch value := parameters[key].(type) { + case int: + return value + case int32: + return int(value) + case int64: + return int(value) + case float64: + return int(value) + default: + return fallback + } +} + +func parameterBool(parameters map[string]interface{}, key string) (bool, bool) { + value, ok := parameters[key].(bool) + return value, ok +} + +func setDefault(config map[string]interface{}, key string, value interface{}) { + if _, exists := config[key]; !exists { + config[key] = value + } +} + +func deviceVolume(name, path string) corev1.Volume { + typeCharDevice := corev1.HostPathCharDev + return corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: path, + Type: &typeCharDevice, + }}, + } +} + +func boolPtr(value bool) *bool { + return &value +} + +func (p *Provisioner) Cleanup( + ctx context.Context, + exporterSet *virtualtargetv1alpha1.ExporterSet, + exporter *jumpstarterdevv1alpha1.Exporter, +) error { + // Cuttlefish state and fetched images are Pod-scoped. Kubernetes removes + // the Pod-owned emptyDir volumes, while a claimed image tree is external + // and must outlive the exporter, so there is nothing for the provisioner + // to clean up here. + return nil +} + +func storageSizes(parameters map[string]interface{}) (resource.Quantity, resource.Quantity, resource.Quantity, error) { + sizes := []resource.Quantity{resource.MustParse("20Gi"), resource.MustParse("20Gi"), resource.MustParse("4Gi")} + raw, exists := parameters["storage"] + if !exists { + return sizes[0], sizes[1], sizes[2], nil + } + storage, ok := raw.(map[string]interface{}) + if !ok { + return sizes[0], sizes[1], sizes[2], fmt.Errorf("parameters.storage must be an object") + } + for i, key := range []string{"imageSize", "stateSize", "tmpSize"} { + if value, exists := storage[key]; exists { + valueString, ok := value.(string) + quantity, err := resource.ParseQuantity(valueString) + if !ok || err != nil || quantity.Sign() <= 0 { + return sizes[0], sizes[1], sizes[2], fmt.Errorf("parameters.storage.%s must be a positive storage quantity", key) + } + sizes[i] = quantity + } + } + return sizes[0], sizes[1], sizes[2], nil +} + +func reserveStorage(resources *corev1.ResourceRequirements, budget resource.Quantity) error { + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { + if value, exists := values[corev1.ResourceEphemeralStorage]; exists && value.Cmp(budget) < 0 { + return fmt.Errorf("ephemeral-storage must be at least %s for Cuttlefish volume budgets and overhead", budget.String()) + } + } + if _, exists := resources.Requests[corev1.ResourceEphemeralStorage]; !exists { + resources.Requests[corev1.ResourceEphemeralStorage] = budget.DeepCopy() + } + if _, exists := resources.Limits[corev1.ResourceEphemeralStorage]; !exists { + resources.Limits[corev1.ResourceEphemeralStorage] = resources.Requests[corev1.ResourceEphemeralStorage].DeepCopy() + } + request := resources.Requests[corev1.ResourceEphemeralStorage] + if request.Cmp(resources.Limits[corev1.ResourceEphemeralStorage]) > 0 { + return fmt.Errorf("ephemeral-storage request exceeds limit") + } + return nil +} diff --git a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go new file mode 100644 index 000000000..a27baae57 --- /dev/null +++ b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go @@ -0,0 +1,434 @@ +package cuttlefish + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestProvisionerName(t *testing.T) { + if got := New("dev").Name(); got != ProvisionerName { + t.Fatalf("Name() = %q, want %q", got, ProvisionerName) + } +} + +func TestRenderPod(t *testing.T) { + exporterSet := testExporterSet() + vtc := &virtualtargetv1alpha1.VirtualTargetClass{ + Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Provisioner: ProvisionerName}, + } + + pod, err := New("dev").RenderPod(context.Background(), exporterSet, vtc, map[string]interface{}{ + "fetch_images": true, + "runtime_privileged": true, + "service_account_name": "cuttlefish-runtime", + }, nil, nil) + if err != nil { + t.Fatal(err) + } + if len(pod.Spec.InitContainers) != 5 { + t.Fatalf("init container count = %d, want 5", len(pod.Spec.InitContainers)) + } + if pod.Spec.InitContainers[0].Name != "fetch-images" { + t.Errorf("first init container = %q", pod.Spec.InitContainers[0].Name) + } + if pod.Spec.InitContainers[2].Name != "cuttlefish" || pod.Spec.InitContainers[2].RestartPolicy == nil { + t.Errorf("runtime sidecar = %#v", pod.Spec.InitContainers[2]) + } + if pod.Spec.InitContainers[2].SecurityContext == nil || + pod.Spec.InitContainers[2].SecurityContext.Privileged == nil || + !*pod.Spec.InitContainers[2].SecurityContext.Privileged { + t.Fatal("Cuttlefish runtime must be privileged") + } + if pod.Spec.InitContainers[3].Name != "cuttlefish-relay" { + t.Errorf("relay container = %q", pod.Spec.InitContainers[3].Name) + } + if len(pod.Spec.Containers) != 1 || pod.Spec.Containers[0].Name != "exporter" { + t.Fatalf("containers = %#v", pod.Spec.Containers) + } + if pod.Spec.Containers[0].Env[0].Name != "HOME" || pod.Spec.Containers[0].Env[0].Value != "/tmp" { + t.Errorf("exporter HOME = %#v, want /tmp", pod.Spec.Containers[0].Env[0]) + } + if !hasVolume(pod.Spec.Volumes, "kvm", "/dev/kvm") || !hasVolume(pod.Spec.Volumes, "tun", "/dev/net/tun") { + t.Fatalf("device volumes missing: %#v", pod.Spec.Volumes) + } +} + +func TestRenderPod_rejectsFetchingIntoClaim(t *testing.T) { + exporterSet := testExporterSet() + vtc := &virtualtargetv1alpha1.VirtualTargetClass{} + + _, err := New("dev").RenderPod(context.Background(), exporterSet, vtc, map[string]interface{}{ + "fetch_images": true, + "image_volume_claim": "cuttlefish-images", + "runtime_privileged": true, + "service_account_name": "cuttlefish-runtime", + }, nil, nil) + if err == nil { + t.Fatal("RenderPod() succeeded; want an error for fetch_images with image_volume_claim") + } +} + +func renderTestPod(t *testing.T, params map[string]interface{}) *corev1.Pod { + t.Helper() + params["runtime_privileged"] = true + params["service_account_name"] = "cuttlefish-runtime" + pod, err := New("dev").RenderPod(context.Background(), testExporterSet(), &virtualtargetv1alpha1.VirtualTargetClass{}, params, nil, nil) + if err != nil { + t.Fatal(err) + } + return pod +} + +func TestRenderPod_privateImageCopy(t *testing.T) { + for _, readOnly := range []bool{true, false} { + pod := renderTestPod(t, map[string]interface{}{"image_volume_claim": "images", "image_volume_read_only": readOnly}) + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && !volume.PersistentVolumeClaim.ReadOnly { + t.Fatal("source claim is writable") + } + if volume.Name == "cvd-images" && volume.EmptyDir == nil { + t.Fatal("missing private image volume") + } + } + copy := pod.Spec.InitContainers[0] + if copy.Name != "copy-images" || !copy.VolumeMounts[0].ReadOnly || copy.VolumeMounts[1].ReadOnly { + t.Fatal("invalid copy mounts") + } + for _, container := range pod.Spec.InitContainers[1:] { + for _, mount := range container.VolumeMounts { + if mount.Name == "image-source" { + t.Fatalf("%s can access shared source", container.Name) + } + } + } + } +} + +func TestRenderPod_healthAndRelayIsolation(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + gate := pod.Spec.InitContainers[len(pod.Spec.InitContainers)-1] + if gate.Name != "wait-for-cuttlefish" || !strings.Contains(gate.Command[2], "127.0.0.1:2081/_debug/statusz") { + t.Fatal("missing API startup gate") + } + probe := pod.Spec.Containers[0].LivenessProbe + if probe == nil || probe.Exec.Command[2] != "jumpstarter_driver_cuttlefish.health" { + t.Fatal("missing runtime failure detection") + } + for _, container := range pod.Spec.InitContainers { + if container.Name == "cuttlefish-relay" && strings.Count(container.Command[2], "bind=127.0.0.1") != 2 { + t.Fatal("relay exposed outside Pod") + } + } +} + +func TestRenderPod_storageBudgets(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true, "storage": map[string]interface{}{"imageSize": "8Gi", "stateSize": "4Gi", "tmpSize": "2Gi"}}) + total := resource.MustParse("1Gi") + for _, volume := range pod.Spec.Volumes { + if volume.EmptyDir != nil { + if volume.EmptyDir.SizeLimit == nil { + t.Fatalf("%s has no budget", volume.Name) + } + total.Add(*volume.EmptyDir.SizeLimit) + } + } + for _, container := range pod.Spec.InitContainers { + if container.Name == "fetch-images" || container.Name == "cuttlefish" { + request := container.Resources.Requests[corev1.ResourceEphemeralStorage] + limit := container.Resources.Limits[corev1.ResourceEphemeralStorage] + if request.Cmp(total) != 0 || limit.Cmp(total) != 0 { + t.Fatalf("%s storage does not cover volumes: %v", container.Name, container.Resources) + } + } + } +} + +func TestStorageValidation(t *testing.T) { + for _, value := range []interface{}{"", "0", "-1Gi", "invalid", 42} { + _, _, _, err := storageSizes(map[string]interface{}{"storage": map[string]interface{}{"imageSize": value}}) + if err == nil { + t.Fatalf("accepted invalid size %v", value) + } + } + budget := resource.MustParse("10Gi") + for _, resources := range []corev1.ResourceRequirements{ + {Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, + {Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, + {Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("20Gi")}, Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: budget}}, + } { + if reserveStorage(&resources, budget) == nil { + t.Fatal("accepted insufficient or inconsistent storage") + } + } +} + +func TestEnrichExporterExport(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + {Name: "cuttlefish", Type: cuttlefishDriverType}, + {Name: "netsim", Type: netsimDriverType}, + {Name: "bt_peer", Type: btPeerDriverType}, + } + result, err := New("dev").EnrichExporterExport(drivers, map[string]interface{}{ + "default_build": "aosp/test", + "gpu_mode": "none", + }) + if err != nil { + t.Fatal(err) + } + + cuttlefish := configFor(t, result[0]) + if cuttlefish["host"] != "127.0.0.1" || cuttlefish["port"] != float64(hostOrchestratorPort) { + t.Errorf("Cuttlefish endpoint = %#v", cuttlefish) + } + envConfig := cuttlefish["env_config"].(map[string]interface{}) + instances := envConfig["instances"].([]interface{}) + instance := instances[0].(map[string]interface{}) + graphics := instance["graphics"].(map[string]interface{}) + if graphics["gpu_mode"] != "none" { + t.Errorf("gpu_mode = %v", graphics["gpu_mode"]) + } + vm := instance["vm"].(map[string]interface{}) + if vm["cpus"] != float64(defaultVMCPUs) || vm["memory_mb"] != float64(defaultVMMemoryMB) { + t.Errorf("vm config = %#v", vm) + } + + netsim := configFor(t, result[1]) + if netsim["host"] != "127.0.0.1" || netsim["port"] != float64(netsimRelayPort) { + t.Errorf("netsim config = %#v", netsim) + } + if _, exists := netsim["transport"]; exists { + t.Error("netsim driver does not accept transport") + } + btPeer := configFor(t, result[2]) + if btPeer["transport"] != fmt.Sprintf("tcp-client:127.0.0.1:%d", hciRelayPort) { + t.Errorf("bt_peer config = %#v", btPeer) + } +} + +func TestEnrichExporterExportDefaultsPodSafeGraphicsAndVM(t *testing.T) { + result, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{ + {Name: "cuttlefish", Type: cuttlefishDriverType}, + }, nil) + if err != nil { + t.Fatal(err) + } + + config := configFor(t, result[0]) + envConfig := config["env_config"].(map[string]interface{}) + instance := envConfig["instances"].([]interface{})[0].(map[string]interface{}) + graphics := instance["graphics"].(map[string]interface{}) + if graphics["gpu_mode"] != defaultGPUMode { + t.Errorf("gpu_mode = %v, want %q", graphics["gpu_mode"], defaultGPUMode) + } +} + +func TestEnrichExporterExportRejectsExternalEndpoints(t *testing.T) { + for _, config := range []map[string]interface{}{{"host": "custom-host"}, {"port": 9999}, {"instance_num": 2}, {"scheme": "https"}} { + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(config)} + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{driver}, nil); err == nil { + t.Errorf("accepted external endpoint %v", config) + } + } +} + +func configFor(t *testing.T, driver virtualtargetv1alpha1.DriverConfig) map[string]interface{} { + t.Helper() + var config map[string]interface{} + if err := json.Unmarshal(driver.Config.Raw, &config); err != nil { + t.Fatal(err) + } + return config +} + +func hasVolume(volumes []corev1.Volume, name, path string) bool { + for _, volume := range volumes { + if volume.Name == name && volume.HostPath != nil && volume.HostPath.Path == path { + return true + } + } + return false +} + +func mustJSON(value interface{}) *apiextensionsv1.JSON { + raw, err := json.Marshal(value) + if err != nil { + panic(err) + } + return &apiextensionsv1.JSON{Raw: raw} +} + +func testExporterSet() *virtualtargetv1alpha1.ExporterSet { + return &virtualtargetv1alpha1.ExporterSet{ + ObjectMeta: metav1.ObjectMeta{Name: "cuttlefish", Namespace: "default", UID: "test-uid"}, + Spec: virtualtargetv1alpha1.ExporterSetSpec{Template: virtualtargetv1alpha1.ExporterSetTemplate{Spec: virtualtargetv1alpha1.ExporterTemplateSpec{Drivers: []virtualtargetv1alpha1.DriverConfig{{Name: "cuttlefish", Type: cuttlefishDriverType}}}}}, + } +} + +func TestRelayPortValidation(t *testing.T) { + for _, params := range []map[string]interface{}{ + {"hci_relay_port": 17681}, {"netsim_relay_port": 7681}, {"hci_relay_port": 7300}, + {"netsim_relay_port": 0}, {"netsim_relay_port": -1}, {"netsim_relay_port": 65536}, + {"netsim_relay_port": 1234.5}, {"netsim_relay_port": "1234"}, {"netsim_relay_port": true}, + {"netsim_relay_port": 80}, {"netsim_relay_port": 19531}, {"hci_relay_port": 6521}, + {"host_orchestrator_port": 9999}, {"netsim_relay_port": 2081}, {"netsim_relay_port": 15550}, + } { + if _, _, err := relayPorts(params); err == nil { + t.Errorf("accepted %v", params) + } + } + if n, h, err := relayPorts(map[string]interface{}{"netsim_relay_port": float64(27681), "hci_relay_port": 27300}); err != nil || n != 27681 || h != 27300 { + t.Fatalf("valid ports: %d %d %v", n, h, err) + } +} + +func TestManagedContract(t *testing.T) { + for _, config := range []map[string]interface{}{ + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{}, map[string]interface{}{}}}}, + {"env_config": map[string]interface{}{"instances": "invalid"}}, + {"env_config": map[string]interface{}{"instances": []interface{}{}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{nil}}}, + {"env_config": "invalid"}, + {"env_config": map[string]interface{}{"netsim_bt": false}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"memory_mb": -1}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"crosvm": map[string]interface{}{"vhost_user_vsock": "false"}}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"qemu": map[string]interface{}{}}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"gem5": map[string]interface{}{}}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"crosvm": map[string]interface{}{"vhost_user_vsock": true}}}}}}, + } { + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(config)} + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{driver}, nil); err == nil { + t.Errorf("accepted %v", config) + } + } + drivers := testExporterSet().Spec.Template.Spec.Drivers + if _, err := New("dev").EnrichExporterExport(nil, nil); err == nil { + t.Fatal("accepted missing Cuttlefish driver") + } + if _, err := New("dev").EnrichExporterExport(append(drivers, drivers[0]), nil); err == nil { + t.Fatal("accepted multiple Cuttlefish drivers") + } + if _, err := New("dev").EnrichExporterExport(drivers, map[string]interface{}{"vm_memory_mb": 12.5}); err == nil { + t.Fatal("accepted fractional VM memory") + } + enriched, err := New("dev").EnrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + config := configFor(t, enriched[0]) + vm := config["env_config"].(map[string]interface{})["instances"].([]interface{})[0].(map[string]interface{})["vm"].(map[string]interface{}) + if config["managed"] != true || vm["crosvm"].(map[string]interface{})["vhost_user_vsock"] != "true" { + t.Fatalf("missing managed isolation: %v", config) + } +} + +func TestRuntimeMemory(t *testing.T) { + for _, tc := range []struct { + name, memory, request, limit, want string + invalid bool + }{ + {name: "default", want: "10Gi"}, + {name: "small limit", limit: "1Gi", invalid: true}, + {name: "small request", request: "8Gi", invalid: true}, + {name: "override", memory: "16384", want: "18Gi"}, + {name: "override limit", memory: "16384", limit: "10Gi", invalid: true}, + {name: "request exceeds limit", request: "12Gi", limit: "10Gi", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + es := testExporterSet() + if tc.memory != "" { + es.Spec.Template.Spec.Drivers[0].Config = &apiextensionsv1.JSON{Raw: []byte(`{"env_config":{"instances":[{"vm":{"memory_mb":` + tc.memory + `}}]}}`)} + } + resources := &corev1.ResourceRequirements{Requests: corev1.ResourceList{}, Limits: corev1.ResourceList{}} + if tc.request != "" { + resources.Requests[corev1.ResourceMemory] = resource.MustParse(tc.request) + } + if tc.limit != "" { + resources.Limits[corev1.ResourceMemory] = resource.MustParse(tc.limit) + } + vtc := &virtualtargetv1alpha1.VirtualTargetClass{Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Scheduling: &virtualtargetv1alpha1.SchedulingSpec{Resources: resources}}} + pod, err := New("dev").RenderPod(context.Background(), es, vtc, map[string]interface{}{"runtime_privileged": true, "service_account_name": "cuttlefish-runtime", "fetch_images": true}, nil, nil) + if tc.invalid { + if err == nil { + t.Fatal("accepted invalid resources") + } + return + } + if err != nil { + t.Fatal(err) + } + for _, c := range pod.Spec.InitContainers { + if c.Name == "cuttlefish" && c.Resources.Requests.Memory().Cmp(resource.MustParse(tc.want)) != 0 { + t.Fatalf("memory = %s, want %s", c.Resources.Requests.Memory(), tc.want) + } + } + }) + } +} + +func TestPodIsolation(t *testing.T) { + es := testExporterSet() + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + policy := New("dev").RenderNetworkPolicy(es) + if policy.Spec.PodSelector.MatchLabels[isolationLabel] != pod.Labels[isolationLabel] || len(policy.Spec.Ingress) != 0 || len(policy.Spec.PolicyTypes) != 1 || policy.Spec.PolicyTypes[0] != "Ingress" { + t.Fatalf("incorrect isolation policy: %#v", policy.Spec) + } + if pod.Spec.ServiceAccountName != "cuttlefish-runtime" || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("workload service account not isolated") + } + for _, volume := range pod.Spec.Volumes { + if volume.HostPath != nil && volume.HostPath.Path == "/dev/vhost-vsock" { + t.Fatal("kernel VSOCK device mounted") + } + } + for _, params := range []map[string]interface{}{ + {"fetch_images": true, "service_account_name": "cuttlefish-runtime", "runtime_privileged": false}, + {"fetch_images": true, "runtime_privileged": true}, + {"fetch_images": true, "runtime_privileged": true, "service_account_name": "default"}, + {"fetch_images": true, "runtime_privileged": true, "service_account_name": "Invalid_Name"}, + } { + if _, err := New("dev").RenderPod(context.Background(), es, &virtualtargetv1alpha1.VirtualTargetClass{}, params, nil, nil); err == nil { + t.Fatalf("accepted %v", params) + } + } +} + +func TestRelaySupervisorExitsWhenEitherRelayFails(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + var command []string + for _, c := range pod.Spec.InitContainers { + if c.Name == "cuttlefish-relay" { + command = c.Command + } + } + for _, failingPort := range []int{netsimRelayPort, hciRelayPort} { + t.Run(fmt.Sprint(failingPort), func(t *testing.T) { + dir := t.TempDir() + script := fmt.Sprintf("#!/bin/sh\ncase \"$1\" in TCP-LISTEN:%d,*) exit 1;; esac\nexec sleep 30\n", failingPort) + if err := os.WriteFile(filepath.Join(dir, "socat"), []byte(script), 0755); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, command[0], command[1:]...) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + cmd.WaitDelay = time.Second + err := cmd.Run() + if err == nil || ctx.Err() != nil { + t.Fatalf("supervisor did not promptly fail: %v, %v", err, ctx.Err()) + } + }) + } +} diff --git a/controller/internal/exporterset/reconciler.go b/controller/internal/exporterset/reconciler.go index 8491e875b..843800655 100644 --- a/controller/internal/exporterset/reconciler.go +++ b/controller/internal/exporterset/reconciler.go @@ -40,6 +40,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -88,6 +89,7 @@ type ExporterSetReconciler struct { LastScaleDownAction map[types.NamespacedName]time.Time } +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/finalizers,verbs=update @@ -173,6 +175,10 @@ func (r *ExporterSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if err := r.syncNetworkPolicy(ctx, &exporterSet); err != nil { + return ctrl.Result{}, err + } + logger.Info("reconciling ExporterSet", "name", exporterSet.Name, "provisioner", vtc.Spec.Provisioner, @@ -288,6 +294,34 @@ func (r *ExporterSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result, nil } +// NetworkPolicyProvisioner isolates backend listeners before any workload Pod is created. +type NetworkPolicyProvisioner interface { + RenderNetworkPolicy(*virtualtargetv1alpha1.ExporterSet) *networkingv1.NetworkPolicy +} + +func (r *ExporterSetReconciler) syncNetworkPolicy(ctx context.Context, es *virtualtargetv1alpha1.ExporterSet) error { + provisioner, ok := r.Provisioner.(NetworkPolicyProvisioner) + if !ok { + return nil + } + desired := provisioner.RenderNetworkPolicy(es) + policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, policy, func() error { + if !policy.CreationTimestamp.IsZero() && !metav1.IsControlledBy(policy, es) { + return fmt.Errorf("network policy %s is not owned by ExporterSet", policy.Name) + } + if err := ctrl.SetControllerReference(es, policy, r.Scheme); err != nil { + return err + } + policy.Spec = *desired.Spec.DeepCopy() + return nil + }) + if err != nil { + return fmt.Errorf("ensure runtime network isolation: %w", err) + } + return nil +} + type poolState struct { replicas int32 ready int32 @@ -1267,7 +1301,7 @@ func filterOwnedExporters( // SetupWithManager sets up the controller with the Manager. func (r *ExporterSetReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + builder := ctrl.NewControllerManagedBy(mgr). For(&virtualtargetv1alpha1.ExporterSet{}). Owns(&jumpstarterdevv1alpha1.Exporter{}). Watches( @@ -1282,8 +1316,11 @@ func (r *ExporterSetReconciler) SetupWithManager(mgr ctrl.Manager) error { &jumpstarterdevv1alpha1.Lease{}, handler.EnqueueRequestsFromMapFunc(r.findExporterSetsForLease), ). - Named("exporterset"). - Complete(r) + Named("exporterset") + if _, ok := r.Provisioner.(NetworkPolicyProvisioner); ok { + builder = builder.Owns(&networkingv1.NetworkPolicy{}) + } + return builder.Complete(r) } // findExporterSetForPod bridges the ExporterSet→Exporter→Pod grandchild gap via label. diff --git a/python/packages/jumpstarter-driver-cuttlefish/README.md b/python/packages/jumpstarter-driver-cuttlefish/README.md index 5f4050108..6a05f5142 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/README.md +++ b/python/packages/jumpstarter-driver-cuttlefish/README.md @@ -6,8 +6,11 @@ virtual devices through the [Host Orchestrator](https://github.com/google/android-cuttlefish) REST API. It provides full CVD (Cuttlefish Virtual Device) lifecycle management through standard Jumpstarter interfaces: `VirtualPowerInterface` for on/off/cycle, -plus cuttlefish-specific operations -(snapshot, powerwash, restart). +plus cuttlefish-specific operations (powerwash and restart). + +For managed Kubernetes exporters, see the [Cuttlefish ExporterSet deployment guide](https://github.com/jumpstarter-dev/jumpstarter/blob/main/controller/internal/exporterset/provisioners/cuttlefish/README.md) +for service accounts/SCCs, network isolation, resource budgets, pinned images, +private VSOCK, storage access modes, and failure recovery. ## Installation @@ -114,11 +117,10 @@ are preserved. ### Teardown -Delete CVDs and snapshots when done to avoid accumulation: +Delete CVDs when done to avoid accumulation: ```bash j power off --destroy # deletes the CVD -j cuttlefish snapshot delete # remove specific snapshots ``` ## Configuration @@ -137,11 +139,8 @@ export: instances: - disk: default_build: /home/vsoc-01/fetch - vm: - enable_virtiofs: false # required for snapshot support common: host_package: /home/vsoc-01/fetch - gpu_mode: guest_swiftshader # required for snapshot support netsim: type: jumpstarter_driver_netsim.driver.Netsim config: @@ -220,10 +219,6 @@ j cuttlefish powerbtn # List running operations j cuttlefish ops -# Snapshot management -# Requires: x86_64 host, enable_virtiofs: false, gpu_mode: guest_swiftshader -j cuttlefish snapshot create --id my-snapshot -j cuttlefish snapshot delete ``` ### Python API @@ -251,9 +246,6 @@ with serve(driver) as client: cvds = client.list_cvds() print(cvds) - # Snapshots - client.create_snapshot(snapshot_id="baseline") - # Cleanup client.power.off(destroy=True) ``` From 1c252ea901ea31f061cfc3cfa93d0ef3b46355a7 Mon Sep 17 00:00:00 2001 From: Benny Zlotnik Date: Thu, 10 Sep 2026 08:32:43 +0300 Subject: [PATCH 3/3] simplify cuttlefish provisioner parameters and rendering Drop the socat relay sidecar: all containers share the Pod network namespace, so the exporter reaches netsim and the HCI listener on loopback directly. This removes relay_image, netsim_relay_port, hci_relay_port and the reserved port table. Drop host_orchestrator_port, which only accepted one value, and image_volume_read_only, which was validated but never read. Read each parameter once, validated, and pass the effective guest size to the resource budget instead of re-decoding the enriched driver config. Pin the managed endpoint and VSOCK/netsim settings through one helper, reserve storage where containers are built, split RenderPod into small helpers, and move the startup gate into `health --wait` instead of an inline Python program. Signed-off-by: Benny Zlotnik --- .../provisioners/cuttlefish/README.md | 80 +- .../provisioners/cuttlefish/cuttlefish.go | 763 ++++++++---------- .../cuttlefish/cuttlefish_test.go | 351 ++++---- .../jumpstarter_driver_cuttlefish/health.py | 15 +- .../health_test.py | 19 +- 5 files changed, 636 insertions(+), 592 deletions(-) diff --git a/controller/internal/exporterset/provisioners/cuttlefish/README.md b/controller/internal/exporterset/provisioners/cuttlefish/README.md index ba7937c00..a54c9c42e 100644 --- a/controller/internal/exporterset/provisioners/cuttlefish/README.md +++ b/controller/internal/exporterset/provisioners/cuttlefish/README.md @@ -65,10 +65,12 @@ service account, runtime marker and managed driver configuration. ## Configuration and resource allocation The provisioner requires exactly one Cuttlefish driver and one -`env_config.instances` entry. It fixes the managed endpoint to -`http://127.0.0.1:2081` and `instance_num` to 1. A different -`host_orchestrator_port` is rejected because the upstream image fixes its service -and nginx upstream configuration to 2081. +`env_config.instances` entry. It pins the managed endpoint to +`http://127.0.0.1:2081` and `instance_num` to 1; a template that sets different +values is rejected. The upstream image fixes Host Orchestrator to 2081 behind +nginx on 2080, and all containers share the Pod network namespace, so the netsim +and bt_peer drivers are pointed directly at the simulators on loopback ports +7681 and 7300. Guest defaults are 4 CPUs and 8192 MiB. Runtime memory requests default to the **effective** guest memory plus `runtime_memory_overhead_mb` (2048 MiB by default). @@ -77,10 +79,6 @@ larger simulator workloads. Guest values in driver `env_config` take precedence over `vm_cpus` and `vm_memory_mb` when calculating the budget. CPU requests default to guest CPUs, or to an explicit CPU limit; CPU overcommit remains configurable. -Relay ports must be distinct integers in 1..65535 and must not collide with the -managed runtime's service ports. Defaults are 17681 (netsim) and 17300 (HCI). -The relay supervisor exits if either listener process dies. - `create_cvd()` in managed mode accepts only the exact configured `env_config`, checks the entire Host Orchestrator inventory, and serializes creation with other lifecycle operations. Destroy the existing CVD before creating another. This @@ -118,6 +116,23 @@ tests cannot establish compatibility of a mutable image tag or guest build. ## Image PVCs and reproducibility +Exactly one image source is required: set either `fetch_images: true` or +`image_volume_claim`. Setting both, or neither, is rejected. Fetching downloads +`default_build` into each Pod; a claim provisions from a prewarmed PVC: + +```yaml +# Fetch per Pod. +parameters: + fetch_images: true + default_build: "/aosp_cf_x86_64_auto-userdebug" +``` + +```yaml +# Prewarmed PVC. default_build is unused; the images come from the claim. +parameters: + image_volume_claim: cuttlefish-images +``` + A prewarmed image PVC is mounted read-only, then copied into a private writable `emptyDir`. It remains a Pod volume after the init container exits. Read-only mounting does **not** change the PVC's access mode or remove attachment constraints. @@ -127,7 +142,7 @@ with the required topology. For ReadWriteOnce, keep readers on one compatible node; ReadWriteOncePod permits only one Pod. Alternatively fetch images per Pod. See [Kubernetes access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). -Use an immutable runtime manifest digest, relay digest and Android build ID for +Use an immutable runtime manifest digest and Android build ID for repeatable replacements. This example is a template: substitute verified values from a tested pair; the placeholders are not a published compatibility claim. Pin the exporter image built with this provisioner/driver change too. @@ -145,7 +160,6 @@ spec: runtime_privileged: true fetch_images: true default_build: "/aosp_cf_x86_64_auto-userdebug" - relay_image: "docker.io/alpine/socat@sha256:" vm_cpus: 4 vm_memory_mb: 8192 runtime_memory_overhead_mb: 2048 @@ -163,6 +177,46 @@ spec: memory: 10Gi ``` +The class carries the provisioner parameters and images; the drivers live in the +ExporterSet template. Use `ExitAndReplace`, and keep `env_config.instances` to a +single entry. The provisioner fills in the managed endpoints, so the driver +configuration below only needs what it overrides: + +```yaml +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: ExporterSet +metadata: + name: cuttlefish + namespace: cuttlefish-lab +spec: + minReplicas: 0 + maxReplicas: 4 + minAvailableReplicas: 1 + recycleStrategy: ExitAndReplace + virtualTargetClassName: cuttlefish + selector: + matchLabels: + device: cuttlefish + template: + metadata: + labels: + device: cuttlefish + spec: + drivers: + - name: cuttlefish + type: jumpstarter_driver_cuttlefish.driver.Cuttlefish + config: + env_config: + instances: + - vm: + cpus: 4 + memory_mb: 8192 + - name: netsim + type: jumpstarter_driver_netsim.driver.Netsim + - name: bt_peer + type: jumpstarter_driver_bt_peer.driver.BtPeer +``` + Record both the resolved runtime image digest and the guest `fetcher_config.json` with validation results. A prewarmed PVC needs the same build provenance. @@ -171,7 +225,7 @@ with validation results. A prewarmed PVC needs the same build provenance. The exporter liveness probe reads state written atomically by the managed driver. It always checks Host Orchestrator availability and a per-start runtime ID. When the guest is expected to run, it also checks the CVD inventory/status and -simulator/relay TCP listeners in the shared network namespace. It inspects +simulator TCP listeners in the shared network namespace. It inspects listeners instead of opening HCI connections that could disturb Bluetooth peers. A listening socket alone does not prove simulator protocol correctness. @@ -186,8 +240,8 @@ lease. Release that lease to let the controller delete and replace the Pod; reacquire a lease for a fresh device. Automatic replacement during an active lease is not attempted. Use `ExitAndReplace` for managed Cuttlefish pools. -Validate recovery on disposable leased Pods by terminating the VMM, netsim, each -relay, and then the runtime sidecar in separate trials. A component may recover +Validate recovery on disposable leased Pods by terminating the VMM, netsim, and +then the runtime sidecar in separate trials. A component may recover within the probe failure window; verify guest and simulator functionality after recovery. For unrecovered failures and runtime sidecar restarts, confirm liveness failure, client disconnect, retention while leased, and replacement after release. Also diff --git a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go index acea04d65..ab41468d7 100644 --- a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go +++ b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go @@ -19,6 +19,7 @@ limitations under the License. package cuttlefish import ( + "bytes" "context" "encoding/json" "fmt" @@ -42,7 +43,6 @@ const ( DefaultExporterImage = "quay.io/jumpstarter-dev/jumpstarter:latest" DefaultRuntimeImage = "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orchestration/cuttlefish-orchestration:stable" - DefaultRelayImage = "docker.io/alpine/socat:latest" exporterConfigPath = "/etc/jumpstarter/exporters/config.yaml" exporterNonRootUID int64 = 65532 @@ -51,39 +51,56 @@ const ( netsimDriverType = "jumpstarter_driver_netsim.driver.Netsim" btPeerDriverType = "jumpstarter_driver_bt_peer.driver.BtPeer" - // The orchestration image reserves 2080 for nginx when running in a Pod; - // Host Orchestrator consequently listens on 2081. Keep this configurable. + // All containers share the Pod network namespace, so the exporter reaches + // Host Orchestrator and the simulators on loopback. The orchestration image + // reserves 2080 for nginx inside a Pod, so Host Orchestrator listens on 2081. hostOrchestratorPort = 2081 - netsimRelayPort = 17681 - hciRelayPort = 17300 - defaultGPUMode = "guest_swiftshader" - defaultVMCPUs = 4 - defaultVMMemoryMB = 8192 - isolationLabel = "cuttlefish.jumpstarter.dev/exporter-set" - healthStatePath = "/tmp/jumpstarter-cuttlefish-health.json" - runtimeIDPath = "/run/cuttlefish-runtime/runtime-id" - - fetchPath = "/home/vsoc-01/fetch" - cvdStatePath = "/var/tmp/cvd" - androidTmpPath = "/tmp/android" + hostOrchestratorURL = "http://127.0.0.1:2081" + netsimPort = 7681 + hciPort = 7300 + + defaultGPUMode = "guest_swiftshader" + defaultVMCPUs = 4 + defaultVMMemoryMB = 8192 + defaultOverheadMB = 2048 + isolationLabel = "cuttlefish.jumpstarter.dev/exporter-set" + healthStatePath = "/tmp/jumpstarter-cuttlefish-health.json" + runtimeIDMount = "/run/cuttlefish-runtime" + runtimeIDPath = runtimeIDMount + "/runtime-id" + + fetchPath = "/home/vsoc-01/fetch" + cvdStatePath = "/var/tmp/cvd" + androidTmpPath = "/tmp/android" + imageSourcePath = "/image-source" + + runtimeContainerName = "cuttlefish" + gateContainerName = "wait-for-cuttlefish" ) +// healthPorts are the simulator listeners the liveness probe expects while a guest runs. +var healthPorts = []int{netsimPort, hciPort} + type Provisioner struct { Version string } type storageConfig struct { - imageClaim string - fetchImages bool - imageSize, stateSize, tmpSize, budget resource.Quantity - build string + imageClaim string + fetchImages bool + build string + imageSize, stateSize, tmpSize resource.Quantity + // budget is the ephemeral storage every container touching the volumes must reserve. + budget resource.Quantity +} + +// guestSpec is the effective guest size after template values override parameters. +type guestSpec struct { + cpus, memoryMB int } -type runtimeConfig struct { - relayImage string - netsimPort, hciPort int - privileged bool - serviceAccount string +type images struct { + exporter, runtime string + exporterPull, runtimePull corev1.PullPolicy } func New(version string) *Provisioner { @@ -119,8 +136,23 @@ func (p *Provisioner) resolveImageSpec(spec *virtualtargetv1alpha1.ImageSpec, de return image, pullPolicy } +func (p *Provisioner) resolveImages(overrides *virtualtargetv1alpha1.ImageOverrides) images { + var exporterSpec, runtimeSpec *virtualtargetv1alpha1.ImageSpec + if overrides != nil { + exporterSpec, runtimeSpec = overrides.Exporter, overrides.Runtime + } + img := images{} + img.exporter, img.exporterPull = p.resolveImageSpec(exporterSpec, DefaultExporterImage) + img.runtime, img.runtimePull = p.resolveImageSpec(runtimeSpec, DefaultRuntimeImage) + return img +} + func resolveStorageConfig(parameters map[string]interface{}) (storageConfig, error) { - config := storageConfig{} + config := storageConfig{ + imageSize: resource.MustParse("20Gi"), + stateSize: resource.MustParse("20Gi"), + tmpSize: resource.MustParse("4Gi"), + } config.imageClaim, _ = parameters["image_volume_claim"].(string) config.fetchImages, _ = parameters["fetch_images"].(bool) if config.imageClaim != "" && config.fetchImages { @@ -129,47 +161,49 @@ func resolveStorageConfig(parameters map[string]interface{}) (storageConfig, err if config.imageClaim == "" && !config.fetchImages { return config, fmt.Errorf("cuttlefish requires image_volume_claim or fetch_images=true") } - if readOnly, _ := parameters["image_volume_read_only"].(bool); readOnly && config.imageClaim == "" { - return config, fmt.Errorf("image_volume_read_only requires image_volume_claim") + if config.fetchImages { + config.build = "aosp-android-latest-release/aosp_cf_x86_64_auto-userdebug" + if value, ok := parameters["default_build"].(string); ok && value != "" { + config.build = value + } } - var err error - config.imageSize, config.stateSize, config.tmpSize, err = storageSizes(parameters) - if err != nil { - return config, err + if raw, exists := parameters["storage"]; exists { + storage, ok := raw.(map[string]interface{}) + if !ok { + return config, fmt.Errorf("parameters.storage must be an object") + } + for key, target := range map[string]*resource.Quantity{ + "imageSize": &config.imageSize, "stateSize": &config.stateSize, "tmpSize": &config.tmpSize, + } { + value, exists := storage[key] + if !exists { + continue + } + text, ok := value.(string) + quantity, err := resource.ParseQuantity(text) + if !ok || err != nil || quantity.Sign() <= 0 { + return config, fmt.Errorf("parameters.storage.%s must be a positive storage quantity", key) + } + *target = quantity + } } config.budget = config.imageSize.DeepCopy() config.budget.Add(config.stateSize) config.budget.Add(config.tmpSize) config.budget.Add(resource.MustParse("1Gi")) // Container layers and logs need space beyond the volume budgets. - if config.fetchImages { - config.build = resolveDefaultBuild(parameters) - } return config, nil } -func resolveRuntimeConfig(parameters map[string]interface{}) (runtimeConfig, error) { - config := runtimeConfig{relayImage: DefaultRelayImage} - if value, ok := parameters["relay_image"].(string); ok && value != "" { - config.relayImage = value - } - - var err error - config.netsimPort, config.hciPort, err = relayPorts(parameters) - if err != nil { - return config, err - } - configured := false - config.privileged, configured = parameterBool(parameters, "runtime_privileged") - if !configured || !config.privileged { - return config, fmt.Errorf("cuttlefish requires runtime_privileged=true; unprivileged device access is not supported") +func resolveServiceAccount(parameters map[string]interface{}) (string, error) { + if privileged, _ := parameters["runtime_privileged"].(bool); !privileged { + return "", fmt.Errorf("cuttlefish requires runtime_privileged=true; unprivileged device access is not supported") } - - config.serviceAccount, _ = parameters["service_account_name"].(string) - if config.serviceAccount == "" || config.serviceAccount == "default" || len(validation.IsDNS1123Subdomain(config.serviceAccount)) != 0 { - return config, fmt.Errorf("service_account_name must name a dedicated workload service account") + name, _ := parameters["service_account_name"].(string) + if name == "" || name == "default" || len(validation.IsDNS1123Subdomain(name)) != 0 { + return "", fmt.Errorf("service_account_name must name a dedicated workload service account") } - return config, nil + return name, nil } func (p *Provisioner) RenderPod( @@ -177,64 +211,109 @@ func (p *Provisioner) RenderPod( exporterSet *virtualtargetv1alpha1.ExporterSet, vtc *virtualtargetv1alpha1.VirtualTargetClass, mergedParameters map[string]interface{}, - images *virtualtargetv1alpha1.ImageOverrides, + overrides *virtualtargetv1alpha1.ImageOverrides, exporter *jumpstarterdevv1alpha1.Exporter, ) (*corev1.Pod, error) { _ = ctx if exporterSet.Spec.RecycleStrategy == virtualtargetv1alpha1.RecycleStrategyInPlaceReuse { return nil, fmt.Errorf("managed Cuttlefish requires ExitAndReplace recycling") } - - var exporterSpec, runtimeSpec *virtualtargetv1alpha1.ImageSpec - if images != nil { - exporterSpec = images.Exporter - runtimeSpec = images.Runtime - } - exporterImage, exporterPullPolicy := p.resolveImageSpec(exporterSpec, DefaultExporterImage) - runtimeImage, runtimePullPolicy := p.resolveImageSpec(runtimeSpec, DefaultRuntimeImage) storage, err := resolveStorageConfig(mergedParameters) if err != nil { return nil, err } - runtime, err := resolveRuntimeConfig(mergedParameters) + serviceAccount, err := resolveServiceAccount(mergedParameters) if err != nil { return nil, err } - drivers, err := p.EnrichExporterExport(exporterSet.Spec.Template.Spec.Drivers, mergedParameters) + // The reconciler persists the enriched drivers itself; rendering needs the + // validation and the effective guest size for the runtime budget. + _, guest, err := enrichDrivers(exporterSet.Spec.Template.Spec.Drivers, mergedParameters) if err != nil { return nil, err } - runtimeResources := corev1.ResourceRequirements{} - if vtc.Spec.Scheduling != nil && vtc.Spec.Scheduling.Resources != nil { - runtimeResources = *vtc.Spec.Scheduling.Resources.DeepCopy() + resources, err := runtimeResources(vtc, guest, mergedParameters) + if err != nil { + return nil, err } - if err := reserveRuntimeResources(&runtimeResources, drivers, mergedParameters); err != nil { + if err := reserveStorage(&resources, storage.budget); err != nil { return nil, err } + img := p.resolveImages(overrides) + + pod := &corev1.Pod{ + ObjectMeta: podMeta(exporterSet, exporter), + Spec: corev1.PodSpec{ + // Never: ExitAndReplace relies on the exporter (main) exit completing the Pod. + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount, + AutomountServiceAccountToken: boolPtr(false), + InitContainers: initContainers(img, storage, resources), + Containers: []corev1.Container{exporterContainer(img)}, + Volumes: volumes(storage), + }, + } + if vtc.Spec.Scheduling != nil { + if vtc.Spec.Scheduling.NodeSelector != nil { + pod.Spec.NodeSelector = maps.Clone(vtc.Spec.Scheduling.NodeSelector) + } + if vtc.Spec.Scheduling.Tolerations != nil { + pod.Spec.Tolerations = append([]corev1.Toleration(nil), vtc.Spec.Scheduling.Tolerations...) + } + } + return pod, nil +} - podMeta := metav1.ObjectMeta{ +func podMeta(exporterSet *virtualtargetv1alpha1.ExporterSet, exporter *jumpstarterdevv1alpha1.Exporter) metav1.ObjectMeta { + meta := metav1.ObjectMeta{ Namespace: exporterSet.Namespace, Labels: maps.Clone(exporterSet.Spec.Template.Metadata.Labels), Annotations: maps.Clone(exporterSet.Spec.Template.Metadata.Annotations), } if exporter != nil { - podMeta.Name = exporter.Name + meta.Name = exporter.Name } else { - podMeta.GenerateName = fmt.Sprintf("%s-", exporterSet.Name) + meta.GenerateName = exporterSet.Name + "-" } - - if podMeta.Labels == nil { - podMeta.Labels = map[string]string{} + if meta.Labels == nil { + meta.Labels = map[string]string{} } - podMeta.Labels[isolationLabel] = string(exporterSet.UID) + meta.Labels[isolationLabel] = string(exporterSet.UID) + return meta +} + +func exporterSecurityContext() *corev1.SecurityContext { + uid := exporterNonRootUID + return &corev1.SecurityContext{RunAsUser: &uid, RunAsNonRoot: boolPtr(true)} +} - runtimeRestart := corev1.ContainerRestartPolicyAlways - runAsRoot := int64(0) - runAsExporter := exporterNonRootUID - runAsNonRoot := true +// exporterContainer runs jmp behind the health wrapper, which records the +// runtime marker before the exporter registers and backs the liveness probe. +func exporterContainer(img images) corev1.Container { + return corev1.Container{ + Name: "exporter", + Image: img.exporter, + ImagePullPolicy: img.exporterPull, + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", "--run-exporter", + healthStatePath, runtimeIDPath, hostOrchestratorURL, exporterConfigPath}, + Env: []corev1.EnvVar{{Name: "HOME", Value: "/tmp"}}, + SecurityContext: exporterSecurityContext(), + VolumeMounts: []corev1.VolumeMount{{Name: "cvd-state", MountPath: runtimeIDMount, ReadOnly: true}}, + // With restartPolicy Never, a failed liveness check ends the exporter and lets ExitAndReplace recycle the Pod. + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", healthStatePath}, + }}, + PeriodSeconds: 10, TimeoutSeconds: 10, FailureThreshold: 6, + }, + } +} - volumeMounts := []corev1.VolumeMount{ - {Name: "cvd-images", MountPath: fetchPath, ReadOnly: false}, +// initContainers stages images, fixes ownership, starts the runtime as a +// native sidecar and gates the exporter on Host Orchestrator readiness. +func initContainers(img images, storage storageConfig, runtime corev1.ResourceRequirements) []corev1.Container { + stateMounts := []corev1.VolumeMount{ + {Name: "cvd-images", MountPath: fetchPath}, {Name: "cvd-state", MountPath: cvdStatePath}, {Name: "android-tmp", MountPath: androidTmpPath}, } @@ -243,316 +322,239 @@ func (p *Provisioner) RenderPod( {Name: "vhost-net", MountPath: "/dev/vhost-net"}, {Name: "tun", MountPath: "/dev/net/tun"}, } + restartAlways := corev1.ContainerRestartPolicyAlways + root := int64(0) - exporterContainer := corev1.Container{ - Name: "exporter", - VolumeMounts: []corev1.VolumeMount{{Name: "cvd-state", MountPath: "/run/cuttlefish-runtime", ReadOnly: true}}, - Image: exporterImage, - ImagePullPolicy: exporterPullPolicy, - Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", "--run-exporter", - healthStatePath, runtimeIDPath, "http://127.0.0.1:2081", exporterConfigPath}, - Env: []corev1.EnvVar{{ - Name: "HOME", - Value: "/tmp", - }}, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsExporter, - RunAsNonRoot: &runAsNonRoot, - }, - } - if exporter != nil { - exporterContainer.Env = append(exporterContainer.Env, corev1.EnvVar{ - Name: "JUMPSTARTER_EXEC_LOG_FIELDS", - Value: fmt.Sprintf("component=exporter,exporter=%s,namespace=%s", exporter.Name, exporter.Namespace), + var containers []corev1.Container + if storage.imageClaim != "" { + containers = append(containers, corev1.Container{ + Name: "copy-images", Image: img.runtime, ImagePullPolicy: img.runtimePull, + Command: []string{"bash", "-ec", "cp -a --reflink=auto " + imageSourcePath + "/. " + fetchPath + "/"}, + Resources: storageReservation(storage.budget), + VolumeMounts: []corev1.VolumeMount{ + {Name: "image-source", MountPath: imageSourcePath, ReadOnly: true}, + {Name: "cvd-images", MountPath: fetchPath}, + }, }) } - - imageVolume := corev1.Volume{Name: "cvd-images", VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &storage.imageSize}, - }} - permissionCommand := "mkdir -p /var/tmp/cvd /tmp/android && chown -R httpcvd:httpcvd /var/tmp/cvd /tmp/android /home/vsoc-01/fetch" - - initContainers := make([]corev1.Container, 0, 4) if storage.fetchImages { - initContainers = append(initContainers, corev1.Container{ - Name: "fetch-images", - Image: runtimeImage, - ImagePullPolicy: runtimePullPolicy, - Command: []string{"cvd", "fetch", "--default_build=" + storage.build, "--target_directory=" + fetchPath}, - VolumeMounts: []corev1.VolumeMount{{Name: "cvd-images", MountPath: fetchPath, ReadOnly: false}}, + containers = append(containers, corev1.Container{ + Name: "fetch-images", Image: img.runtime, ImagePullPolicy: img.runtimePull, + Command: []string{"cvd", "fetch", "--default_build=" + storage.build, "--target_directory=" + fetchPath}, + Resources: storageReservation(storage.budget), + VolumeMounts: []corev1.VolumeMount{{Name: "cvd-images", MountPath: fetchPath}}, }) } - initContainers = append(initContainers, + return append(containers, corev1.Container{ - Name: "fix-cuttlefish-permissions", - Image: runtimeImage, - ImagePullPolicy: runtimePullPolicy, - Command: []string{ - "bash", "-c", permissionCommand, - }, - VolumeMounts: volumeMounts, + Name: "fix-cuttlefish-permissions", Image: img.runtime, ImagePullPolicy: img.runtimePull, + Command: []string{"bash", "-c", "mkdir -p " + cvdStatePath + " " + androidTmpPath + + " && chown -R httpcvd:httpcvd " + cvdStatePath + " " + androidTmpPath + " " + fetchPath}, + // chown needs UID 0 regardless of the runtime image's default user. + SecurityContext: &corev1.SecurityContext{RunAsUser: &root}, + VolumeMounts: stateMounts, }, corev1.Container{ - Name: "cuttlefish", - Image: runtimeImage, - ImagePullPolicy: runtimePullPolicy, - RestartPolicy: &runtimeRestart, - Command: []string{"bash", "-ec", `cat /proc/sys/kernel/random/uuid > /var/tmp/cvd/runtime-id -chmod 644 /var/tmp/cvd/runtime-id -exec /root/run_services.sh`}, - Resources: runtimeResources, - SecurityContext: &corev1.SecurityContext{Privileged: boolPtr(runtime.privileged), RunAsUser: &runAsRoot}, - VolumeMounts: slices.Concat(volumeMounts, deviceMounts), + Name: runtimeContainerName, Image: img.runtime, ImagePullPolicy: img.runtimePull, + RestartPolicy: &restartAlways, + // The marker lets the exporter and probe detect a sidecar restart that lost runtime state. + Command: []string{"bash", "-ec", "cat /proc/sys/kernel/random/uuid > " + cvdStatePath + "/runtime-id\n" + + "chmod 644 " + cvdStatePath + "/runtime-id\nexec /root/run_services.sh"}, + Resources: runtime, + SecurityContext: &corev1.SecurityContext{Privileged: boolPtr(true), RunAsUser: &root}, + VolumeMounts: append(stateMounts, deviceMounts...), }, corev1.Container{ - Name: "cuttlefish-relay", - Image: runtime.relayImage, - ImagePullPolicy: corev1.PullIfNotPresent, - RestartPolicy: &runtimeRestart, - Command: []string{ - "sh", "-c", - fmt.Sprintf( - `socat TCP-LISTEN:%d,bind=127.0.0.1,fork,reuseaddr TCP:127.0.0.1:7681 & -first=$! -socat TCP-LISTEN:%d,bind=127.0.0.1,fork,reuseaddr TCP:127.0.0.1:7300 & -second=$! -trap 'kill $first $second 2>/dev/null || true' EXIT -while kill -0 $first && kill -0 $second; do sleep 1; done -exit 1`, - runtime.netsimPort, runtime.hciPort, - ), - }, + // Runs in the exporter image so the check shares the network namespace and Python runtime with jmp. + Name: gateContainerName, Image: img.exporter, ImagePullPolicy: img.exporterPull, + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", "--wait", hostOrchestratorURL}, + SecurityContext: exporterSecurityContext(), }, ) +} - pod := &corev1.Pod{ - ObjectMeta: podMeta, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: runtime.serviceAccount, - AutomountServiceAccountToken: boolPtr(false), - InitContainers: initContainers, - Containers: []corev1.Container{exporterContainer}, - Volumes: []corev1.Volume{ - imageVolume, - {Name: "cvd-state", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &storage.stateSize}}}, - {Name: "android-tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &storage.tmpSize}}}, - deviceVolume("kvm", "/dev/kvm"), - deviceVolume("vhost-net", "/dev/vhost-net"), - deviceVolume("tun", "/dev/net/tun"), - }, - }, +func volumes(storage storageConfig) []corev1.Volume { + emptyDir := func(name string, size *resource.Quantity) corev1.Volume { + return corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: size}, + }} } - - if vtc.Spec.Scheduling != nil { - if vtc.Spec.Scheduling.NodeSelector != nil { - pod.Spec.NodeSelector = maps.Clone(vtc.Spec.Scheduling.NodeSelector) - } - if vtc.Spec.Scheduling.Tolerations != nil { - pod.Spec.Tolerations = append([]corev1.Toleration(nil), vtc.Spec.Scheduling.Tolerations...) - } + result := []corev1.Volume{ + emptyDir("cvd-images", &storage.imageSize), + emptyDir("cvd-state", &storage.stateSize), + emptyDir("android-tmp", &storage.tmpSize), + deviceVolume("kvm", "/dev/kvm"), + deviceVolume("vhost-net", "/dev/vhost-net"), + deviceVolume("tun", "/dev/net/tun"), } - if storage.imageClaim != "" { - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{Name: "image-source", VolumeSource: corev1.VolumeSource{ + // The claim is only ever read; each Pod copies it into its private image volume. + result = append(result, corev1.Volume{Name: "image-source", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: storage.imageClaim, ReadOnly: true}, }}) - copyImages := corev1.Container{ - Name: "copy-images", Image: runtimeImage, ImagePullPolicy: runtimePullPolicy, - Command: []string{"bash", "-ec", "cp -a --reflink=auto /image-source/. /home/vsoc-01/fetch/"}, - VolumeMounts: []corev1.VolumeMount{ - {Name: "image-source", MountPath: "/image-source", ReadOnly: true}, - {Name: "cvd-images", MountPath: fetchPath}, - }, - } - pod.Spec.InitContainers = append([]corev1.Container{copyImages}, pod.Spec.InitContainers...) - } - for i := range pod.Spec.InitContainers { - container := &pod.Spec.InitContainers[i] - if container.Name == "cuttlefish" || container.Name == "fetch-images" || container.Name == "copy-images" { - if err := reserveStorage(&container.Resources, storage.budget); err != nil { - return nil, err - } - } } - // Run in the exporter image so the check uses the same network namespace and Python runtime as jmp. - healthURL := fmt.Sprintf("http://127.0.0.1:%d/_debug/statusz", parameterInt(mergedParameters, "host_orchestrator_port", hostOrchestratorPort)) - healthCheck := "import urllib.request; urllib.request.urlopen(" + fmt.Sprintf("%q", healthURL) + ", timeout=3).close()" - pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ - Name: "wait-for-cuttlefish", Image: exporterImage, ImagePullPolicy: exporterPullPolicy, - SecurityContext: exporterContainer.SecurityContext.DeepCopy(), - Command: []string{"python3", "-c", "import time, urllib.request\nfor attempt in range(60):\n try:\n " + healthCheck + "\n break\n except Exception:\n time.sleep(5)\nelse:\n raise SystemExit('Host Orchestrator did not become ready')"}, - }) - // With restartPolicy Never, a failed liveness check ends the exporter and lets ExitAndReplace recycle the Pod. - pod.Spec.Containers[0].LivenessProbe = &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", healthStatePath}}}, - PeriodSeconds: 10, TimeoutSeconds: 10, FailureThreshold: 6, - } - - return pod, nil + return result } func (p *Provisioner) EnrichExporterExport( drivers []virtualtargetv1alpha1.DriverConfig, mergedParameters map[string]interface{}, ) ([]virtualtargetv1alpha1.DriverConfig, error) { - netsimPort, hciPort, err := relayPorts(mergedParameters) - if err != nil { - return nil, err - } - count := 0 - for _, driver := range drivers { - if driver.Type == cuttlefishDriverType { - count++ - } - } - if count != 1 { - return nil, fmt.Errorf("cuttlefish requires exactly one Cuttlefish driver per Pod, got %d", count) - } + enriched, _, err := enrichDrivers(drivers, mergedParameters) + return enriched, err +} + +// enrichDrivers pins every driver to the in-Pod runtime and returns the +// effective guest size the runtime container must budget for. +func enrichDrivers(drivers []virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) ([]virtualtargetv1alpha1.DriverConfig, guestSpec, error) { + var guest guestSpec + found := 0 result := make([]virtualtargetv1alpha1.DriverConfig, 0, len(drivers)) for _, driver := range drivers { var err error switch driver.Type { case cuttlefishDriverType: - driver, err = enrichCuttlefishDriver(driver, mergedParameters) + found++ + driver, guest, err = enrichCuttlefishDriver(driver, parameters) case netsimDriverType: - driver, err = enrichDriverConfig(driver, map[string]interface{}{ - "host": "127.0.0.1", - "port": netsimPort, - }, "netsim") + driver, err = pinDriverConfig(driver, map[string]interface{}{"host": "127.0.0.1", "port": netsimPort}, "netsim") case btPeerDriverType: - driver, err = enrichDriverConfig(driver, map[string]interface{}{ - "transport": fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort), - }, "bt_peer") + driver, err = pinDriverConfig(driver, map[string]interface{}{"transport": fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort)}, "bt_peer") } if err != nil { - return nil, err + return nil, guest, err } result = append(result, driver) } - return result, nil + if found != 1 { + return nil, guest, fmt.Errorf("cuttlefish requires exactly one Cuttlefish driver per Pod, got %d", found) + } + return result, guest, nil } -func enrichCuttlefishDriver(driver virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) (virtualtargetv1alpha1.DriverConfig, error) { - for _, item := range []struct { - key string - fallback int - }{{"vm_cpus", defaultVMCPUs}, {"vm_memory_mb", defaultVMMemoryMB}} { - if _, err := positiveInt(parameters, item.key, item.fallback); err != nil { - return driver, err - } +func enrichCuttlefishDriver(driver virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) (virtualtargetv1alpha1.DriverConfig, guestSpec, error) { + var guest guestSpec + var err error + if guest.cpus, err = positiveInt(parameters, "vm_cpus", defaultVMCPUs); err != nil { + return driver, guest, err + } + if guest.memoryMB, err = positiveInt(parameters, "vm_memory_mb", defaultVMMemoryMB); err != nil { + return driver, guest, err } config, err := decodeConfig(driver, "Cuttlefish") if err != nil { - return driver, err + return driver, guest, err } config["managed"] = true config["health_state_path"] = healthStatePath config["runtime_id_path"] = runtimeIDPath - config["health_ports"] = []int{7681, 7300, parameterInt(parameters, "netsim_relay_port", netsimRelayPort), parameterInt(parameters, "hci_relay_port", hciRelayPort)} - setDefault(config, "scheme", "http") - setDefault(config, "host", "127.0.0.1") - setDefault(config, "port", parameterInt(parameters, "host_orchestrator_port", hostOrchestratorPort)) + config["health_ports"] = healthPorts + // Any other endpoint would bypass the managed runtime in this Pod. + for key, value := range map[string]interface{}{"scheme": "http", "host": "127.0.0.1", "port": hostOrchestratorPort, "instance_num": 1} { + if err := pin(config, key, value); err != nil { + return driver, guest, err + } + } setDefault(config, "group", "cvd") setDefault(config, "name", "1") - setDefault(config, "instance_num", 1) setDefault(config, "boot_timeout", 300) - if err := validateManagedEndpoint(config); err != nil { - return driver, err - } envConfig, err := configObject(config, "env_config") if err != nil { - return driver, err + return driver, guest, err } common, err := configObject(envConfig, "common") if err != nil { - return driver, err + return driver, guest, err } setDefault(common, "host_package", fetchPath) envConfig["common"] = common + // Standalone RootCanal does not propagate the userspace VSOCK flag. + if err := pin(envConfig, "netsim_bt", true); err != nil { + return driver, guest, fmt.Errorf("%w; standalone RootCanal is not supported", err) + } + instances, ok := envConfig["instances"].([]interface{}) if raw, exists := envConfig["instances"]; exists && (!ok || len(instances) != 1) { - return driver, fmt.Errorf("env_config.instances must contain exactly one instance, got %v", raw) + return driver, guest, fmt.Errorf("env_config.instances must contain exactly one instance, got %v", raw) } if len(instances) == 0 { instances = []interface{}{map[string]interface{}{}} } instance, ok := instances[0].(map[string]interface{}) if !ok || instance == nil { - return driver, fmt.Errorf("env_config.instances[0] must be an object") + return driver, guest, fmt.Errorf("env_config.instances[0] must be an object") } disk, err := configObject(instance, "disk") if err != nil { - return driver, err + return driver, guest, err } setDefault(disk, "default_build", fetchPath) instance["disk"] = disk graphics, err := configObject(instance, "graphics") if err != nil { - return driver, err + return driver, guest, err } gpuMode := defaultGPUMode - if configuredGPU, ok := parameters["gpu_mode"].(string); ok && configuredGPU != "" { - gpuMode = configuredGPU + if configured, ok := parameters["gpu_mode"].(string); ok && configured != "" { + gpuMode = configured } setDefault(graphics, "gpu_mode", gpuMode) instance["graphics"] = graphics + vm, err := configObject(instance, "vm") if err != nil { - return driver, err + return driver, guest, err } - if _, exists := vm["qemu"]; exists { - return driver, fmt.Errorf("managed Cuttlefish requires crosvm with private userspace VSOCK") - } - if _, exists := vm["gem5"]; exists { - return driver, fmt.Errorf("managed Cuttlefish requires crosvm with private userspace VSOCK") + for _, other := range []string{"qemu", "gem5"} { + if _, exists := vm[other]; exists { + return driver, guest, fmt.Errorf("managed Cuttlefish requires crosvm with private userspace VSOCK, got vm.%s", other) + } } crosvm, err := configObject(vm, "crosvm") if err != nil { - return driver, err + return driver, guest, err } - if value, exists := crosvm["vhost_user_vsock"]; exists && value != "true" { - return driver, fmt.Errorf("vm.crosvm.vhost_user_vsock must be the string true") + // The upstream schema wants the string "true" here; two Pods on one node share guest CIDs otherwise. + if err := pin(crosvm, "vhost_user_vsock", "true"); err != nil { + return driver, guest, err } - crosvm["vhost_user_vsock"] = "true" vm["crosvm"] = crosvm - if value, exists := envConfig["netsim_bt"]; exists && value != true { - return driver, fmt.Errorf("managed Cuttlefish requires netsim_bt=true; standalone RootCanal is not supported") + // Template guest values take precedence over the class parameters. + setDefault(vm, "cpus", guest.cpus) + setDefault(vm, "memory_mb", guest.memoryMB) + if guest.cpus, err = positiveInt(vm, "cpus", guest.cpus); err != nil { + return driver, guest, err } - envConfig["netsim_bt"] = true - setDefault(vm, "cpus", parameterInt(parameters, "vm_cpus", defaultVMCPUs)) - setDefault(vm, "memory_mb", parameterInt(parameters, "vm_memory_mb", defaultVMMemoryMB)) - if _, err := positiveInt(vm, "cpus", defaultVMCPUs); err != nil { - return driver, err - } - if _, err := positiveInt(vm, "memory_mb", defaultVMMemoryMB); err != nil { - return driver, err + if guest.memoryMB, err = positiveInt(vm, "memory_mb", guest.memoryMB); err != nil { + return driver, guest, err } instance["vm"] = vm instances[0] = instance envConfig["instances"] = instances config["env_config"] = envConfig - return encodeConfig(driver, config) + driver, err = encodeConfig(driver, config) + return driver, guest, err } -func validateManagedEndpoint(config map[string]interface{}) error { - for key, required := range map[string]interface{}{"scheme": "http", "host": "127.0.0.1"} { - if config[key] != required { - return fmt.Errorf("managed Cuttlefish requires %s=%v", key, required) - } - } - if port, err := positiveInt(config, "port", hostOrchestratorPort); err != nil || port != hostOrchestratorPort { - return fmt.Errorf("managed Cuttlefish requires port=%d", hostOrchestratorPort) - } - if instanceNum, err := positiveInt(config, "instance_num", 1); err != nil || instanceNum != 1 { - return fmt.Errorf("managed Cuttlefish requires instance_num=1") +// pin sets config[key] to value and rejects a template value that differs. +// Values are compared through their JSON encoding so 2081 matches 2081.0. +func pin(config map[string]interface{}, key string, value interface{}) error { + if current, exists := config[key]; exists && !jsonEqual(current, value) { + return fmt.Errorf("managed Cuttlefish requires %s=%v, got %v", key, value, current) } + config[key] = value return nil } +func jsonEqual(a, b interface{}) bool { + rawA, errA := json.Marshal(a) + rawB, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(rawA, rawB) +} + func configObject(parent map[string]interface{}, key string) (map[string]interface{}, error) { raw, exists := parent[key] if !exists { @@ -565,13 +567,19 @@ func configObject(parent map[string]interface{}, key string) (map[string]interfa return value, nil } -func enrichDriverConfig(driver virtualtargetv1alpha1.DriverConfig, defaults map[string]interface{}, name string) (virtualtargetv1alpha1.DriverConfig, error) { +// pinDriverConfig pins a sidecar driver to the simulator endpoints this Pod +// runs; a template value that differs points the driver outside the Pod and is +// rejected, while a matching one is preserved. Keys are applied in sorted order +// so a conflicting template yields a stable error. +func pinDriverConfig(driver virtualtargetv1alpha1.DriverConfig, managed map[string]interface{}, name string) (virtualtargetv1alpha1.DriverConfig, error) { config, err := decodeConfig(driver, name) if err != nil { return driver, err } - for key, value := range defaults { - setDefault(config, key, value) + for _, key := range slices.Sorted(maps.Keys(managed)) { + if err := pin(config, key, managed[key]); err != nil { + return driver, fmt.Errorf("%s driver: %w", name, err) + } } return encodeConfig(driver, config) } @@ -598,33 +606,6 @@ func encodeConfig(driver virtualtargetv1alpha1.DriverConfig, config map[string]i return driver, nil } -func resolveDefaultBuild(parameters map[string]interface{}) string { - if value, ok := parameters["default_build"].(string); ok && value != "" { - return value - } - return "aosp-android-latest-release/aosp_cf_x86_64_auto-userdebug" -} - -func relayPorts(parameters map[string]interface{}) (int, int, error) { - ports := []int{netsimRelayPort, hciRelayPort} - reserved := map[int]bool{ - 80: true, 443: true, 1080: true, 1443: true, 2080: true, 2081: true, 2443: true, - 7300: true, 7301: true, 7302: true, 7303: true, 7681: true, 15037: true, 19531: true, - } - if port, err := positiveInt(parameters, "host_orchestrator_port", hostOrchestratorPort); err != nil || port != hostOrchestratorPort { - return 0, 0, fmt.Errorf("host_orchestrator_port must be %d for the orchestration image", hostOrchestratorPort) - } - for i, key := range []string{"netsim_relay_port", "hci_relay_port"} { - port, err := positiveInt(parameters, key, ports[i]) - if err != nil || port > 65535 || reserved[port] || (port >= 6520 && port <= 6620) || (port >= 15550 && port <= 15560) { - return 0, 0, fmt.Errorf("%s must be an integer port in 1..65535 that does not conflict with a runtime service", key) - } - ports[i] = port - reserved[port] = true - } - return ports[0], ports[1], nil -} - func positiveInt(values map[string]interface{}, key string, fallback int) (int, error) { raw, exists := values[key] if !exists { @@ -649,58 +630,78 @@ func positiveInt(values map[string]interface{}, key string, fallback int) (int, return int(value), nil } -func reserveRuntimeResources(resources *corev1.ResourceRequirements, drivers []virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) error { - var vm map[string]interface{} - for _, driver := range drivers { - if driver.Type != cuttlefishDriverType { - continue - } - config, err := decodeConfig(driver, "Cuttlefish") - if err != nil { - return err - } - vm = config["env_config"].(map[string]interface{})["instances"].([]interface{})[0].(map[string]interface{})["vm"].(map[string]interface{}) - } - memory, err := positiveInt(vm, "memory_mb", defaultVMMemoryMB) - if err != nil { - return err - } - cpus, err := positiveInt(vm, "cpus", defaultVMCPUs) - if err != nil { - return err +// runtimeResources starts from the class scheduling resources and guarantees +// the runtime container requests the guest memory plus runtime overhead. +func runtimeResources(vtc *virtualtargetv1alpha1.VirtualTargetClass, guest guestSpec, parameters map[string]interface{}) (corev1.ResourceRequirements, error) { + resources := corev1.ResourceRequirements{} + if vtc.Spec.Scheduling != nil && vtc.Spec.Scheduling.Resources != nil { + resources = *vtc.Spec.Scheduling.Resources.DeepCopy() } - overhead, err := positiveInt(parameters, "runtime_memory_overhead_mb", 2048) + overhead, err := positiveInt(parameters, "runtime_memory_overhead_mb", defaultOverheadMB) if err != nil { - return err + return resources, err } - budget := *resource.NewQuantity((int64(memory)+int64(overhead))*1024*1024, resource.BinarySI) + budget := *resource.NewQuantity(int64(guest.memoryMB+overhead)*1024*1024, resource.BinarySI) if resources.Requests == nil { resources.Requests = corev1.ResourceList{} } for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { if value, exists := values[corev1.ResourceMemory]; exists && value.Cmp(budget) < 0 { - return fmt.Errorf("runtime memory must be at least %s for guest plus overhead", budget.String()) + return resources, fmt.Errorf("runtime memory must be at least %s for guest plus overhead", budget.String()) } } if _, exists := resources.Requests[corev1.ResourceMemory]; !exists { resources.Requests[corev1.ResourceMemory] = budget } if limit, exists := resources.Limits[corev1.ResourceMemory]; exists && resources.Requests.Memory().Cmp(limit) > 0 { - return fmt.Errorf("runtime memory request exceeds limit") + return resources, fmt.Errorf("runtime memory request exceeds limit") } if _, exists := resources.Requests[corev1.ResourceCPU]; !exists { if limit, exists := resources.Limits[corev1.ResourceCPU]; exists { resources.Requests[corev1.ResourceCPU] = limit.DeepCopy() } else { - resources.Requests[corev1.ResourceCPU] = *resource.NewQuantity(int64(cpus), resource.DecimalSI) + resources.Requests[corev1.ResourceCPU] = *resource.NewQuantity(int64(guest.cpus), resource.DecimalSI) } } - if resources.Requests.Cpu().Sign() <= 0 { - return fmt.Errorf("runtime CPU request must be positive") + return resources, fmt.Errorf("runtime CPU request must be positive") } if limit, exists := resources.Limits[corev1.ResourceCPU]; exists && resources.Requests.Cpu().Cmp(limit) > 0 { - return fmt.Errorf("runtime CPU request exceeds limit") + return resources, fmt.Errorf("runtime CPU request exceeds limit") + } + return resources, nil +} + +// storageReservation is the ephemeral storage for containers that only touch the volumes. +func storageReservation(budget resource.Quantity) corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: budget.DeepCopy()}, + Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: budget.DeepCopy()}, + } +} + +// reserveStorage adds the ephemeral storage budget to class-provided resources. +func reserveStorage(resources *corev1.ResourceRequirements, budget resource.Quantity) error { + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { + if value, exists := values[corev1.ResourceEphemeralStorage]; exists && value.Cmp(budget) < 0 { + return fmt.Errorf("ephemeral-storage must be at least %s for Cuttlefish volume budgets and overhead", budget.String()) + } + } + if _, exists := resources.Requests[corev1.ResourceEphemeralStorage]; !exists { + resources.Requests[corev1.ResourceEphemeralStorage] = budget.DeepCopy() + } + if _, exists := resources.Limits[corev1.ResourceEphemeralStorage]; !exists { + resources.Limits[corev1.ResourceEphemeralStorage] = resources.Requests[corev1.ResourceEphemeralStorage].DeepCopy() + } + request := resources.Requests[corev1.ResourceEphemeralStorage] + if request.Cmp(resources.Limits[corev1.ResourceEphemeralStorage]) > 0 { + return fmt.Errorf("ephemeral-storage request exceeds limit") } return nil } @@ -715,26 +716,6 @@ func (p *Provisioner) RenderNetworkPolicy(es *virtualtargetv1alpha1.ExporterSet) } } -func parameterInt(parameters map[string]interface{}, key string, fallback int) int { - switch value := parameters[key].(type) { - case int: - return value - case int32: - return int(value) - case int64: - return int(value) - case float64: - return int(value) - default: - return fallback - } -} - -func parameterBool(parameters map[string]interface{}, key string) (bool, bool) { - value, ok := parameters[key].(bool) - return value, ok -} - func setDefault(config map[string]interface{}, key string, value interface{}) { if _, exists := config[key]; !exists { config[key] = value @@ -767,51 +748,3 @@ func (p *Provisioner) Cleanup( // to clean up here. return nil } - -func storageSizes(parameters map[string]interface{}) (resource.Quantity, resource.Quantity, resource.Quantity, error) { - sizes := []resource.Quantity{resource.MustParse("20Gi"), resource.MustParse("20Gi"), resource.MustParse("4Gi")} - raw, exists := parameters["storage"] - if !exists { - return sizes[0], sizes[1], sizes[2], nil - } - storage, ok := raw.(map[string]interface{}) - if !ok { - return sizes[0], sizes[1], sizes[2], fmt.Errorf("parameters.storage must be an object") - } - for i, key := range []string{"imageSize", "stateSize", "tmpSize"} { - if value, exists := storage[key]; exists { - valueString, ok := value.(string) - quantity, err := resource.ParseQuantity(valueString) - if !ok || err != nil || quantity.Sign() <= 0 { - return sizes[0], sizes[1], sizes[2], fmt.Errorf("parameters.storage.%s must be a positive storage quantity", key) - } - sizes[i] = quantity - } - } - return sizes[0], sizes[1], sizes[2], nil -} - -func reserveStorage(resources *corev1.ResourceRequirements, budget resource.Quantity) error { - if resources.Requests == nil { - resources.Requests = corev1.ResourceList{} - } - if resources.Limits == nil { - resources.Limits = corev1.ResourceList{} - } - for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { - if value, exists := values[corev1.ResourceEphemeralStorage]; exists && value.Cmp(budget) < 0 { - return fmt.Errorf("ephemeral-storage must be at least %s for Cuttlefish volume budgets and overhead", budget.String()) - } - } - if _, exists := resources.Requests[corev1.ResourceEphemeralStorage]; !exists { - resources.Requests[corev1.ResourceEphemeralStorage] = budget.DeepCopy() - } - if _, exists := resources.Limits[corev1.ResourceEphemeralStorage]; !exists { - resources.Limits[corev1.ResourceEphemeralStorage] = resources.Requests[corev1.ResourceEphemeralStorage].DeepCopy() - } - request := resources.Requests[corev1.ResourceEphemeralStorage] - if request.Cmp(resources.Limits[corev1.ResourceEphemeralStorage]) > 0 { - return fmt.Errorf("ephemeral-storage request exceeds limit") - } - return nil -} diff --git a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go index a27baae57..a4306f780 100644 --- a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go +++ b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go @@ -4,12 +4,7 @@ import ( "context" "encoding/json" "fmt" - "os" - "os/exec" - "path/filepath" - "strings" "testing" - "time" virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -25,41 +20,38 @@ func TestProvisionerName(t *testing.T) { } func TestRenderPod(t *testing.T) { - exporterSet := testExporterSet() - vtc := &virtualtargetv1alpha1.VirtualTargetClass{ - Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Provisioner: ProvisionerName}, - } - - pod, err := New("dev").RenderPod(context.Background(), exporterSet, vtc, map[string]interface{}{ - "fetch_images": true, - "runtime_privileged": true, - "service_account_name": "cuttlefish-runtime", - }, nil, nil) - if err != nil { - t.Fatal(err) - } - if len(pod.Spec.InitContainers) != 5 { - t.Fatalf("init container count = %d, want 5", len(pod.Spec.InitContainers)) + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + names := make([]string, 0, len(pod.Spec.InitContainers)) + for _, container := range pod.Spec.InitContainers { + names = append(names, container.Name) } - if pod.Spec.InitContainers[0].Name != "fetch-images" { - t.Errorf("first init container = %q", pod.Spec.InitContainers[0].Name) + want := []string{"fetch-images", "fix-cuttlefish-permissions", runtimeContainerName, gateContainerName} + if fmt.Sprint(names) != fmt.Sprint(want) { + t.Fatalf("init containers = %v, want %v", names, want) } - if pod.Spec.InitContainers[2].Name != "cuttlefish" || pod.Spec.InitContainers[2].RestartPolicy == nil { - t.Errorf("runtime sidecar = %#v", pod.Spec.InitContainers[2]) + runtime := initContainer(t, pod, runtimeContainerName) + if runtime.RestartPolicy == nil || *runtime.RestartPolicy != corev1.ContainerRestartPolicyAlways { + t.Fatal("runtime must be a native sidecar") } - if pod.Spec.InitContainers[2].SecurityContext == nil || - pod.Spec.InitContainers[2].SecurityContext.Privileged == nil || - !*pod.Spec.InitContainers[2].SecurityContext.Privileged { + if runtime.SecurityContext == nil || runtime.SecurityContext.Privileged == nil || !*runtime.SecurityContext.Privileged { t.Fatal("Cuttlefish runtime must be privileged") } - if pod.Spec.InitContainers[3].Name != "cuttlefish-relay" { - t.Errorf("relay container = %q", pod.Spec.InitContainers[3].Name) + permissions := initContainer(t, pod, "fix-cuttlefish-permissions") + if permissions.SecurityContext == nil || permissions.SecurityContext.RunAsUser == nil || *permissions.SecurityContext.RunAsUser != 0 { + t.Fatal("chown init container must run as UID 0") } if len(pod.Spec.Containers) != 1 || pod.Spec.Containers[0].Name != "exporter" { t.Fatalf("containers = %#v", pod.Spec.Containers) } - if pod.Spec.Containers[0].Env[0].Name != "HOME" || pod.Spec.Containers[0].Env[0].Value != "/tmp" { - t.Errorf("exporter HOME = %#v, want /tmp", pod.Spec.Containers[0].Env[0]) + exporter := pod.Spec.Containers[0] + if !hasEnv(exporter.Env, "HOME", "/tmp") { + t.Errorf("exporter HOME = %#v, want /tmp", exporter.Env) + } + if exporter.Command[6] != hostOrchestratorURL { + t.Errorf("exporter endpoint = %q", exporter.Command[6]) + } + if pod.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Error("Pod must not restart the exporter in place") } if !hasVolume(pod.Spec.Volumes, "kvm", "/dev/kvm") || !hasVolume(pod.Spec.Volumes, "tun", "/dev/net/tun") { t.Fatalf("device volumes missing: %#v", pod.Spec.Volumes) @@ -67,10 +59,7 @@ func TestRenderPod(t *testing.T) { } func TestRenderPod_rejectsFetchingIntoClaim(t *testing.T) { - exporterSet := testExporterSet() - vtc := &virtualtargetv1alpha1.VirtualTargetClass{} - - _, err := New("dev").RenderPod(context.Background(), exporterSet, vtc, map[string]interface{}{ + _, err := New("dev").RenderPod(context.Background(), testExporterSet(), &virtualtargetv1alpha1.VirtualTargetClass{}, map[string]interface{}{ "fetch_images": true, "image_volume_claim": "cuttlefish-images", "runtime_privileged": true, @@ -81,56 +70,44 @@ func TestRenderPod_rejectsFetchingIntoClaim(t *testing.T) { } } -func renderTestPod(t *testing.T, params map[string]interface{}) *corev1.Pod { - t.Helper() - params["runtime_privileged"] = true - params["service_account_name"] = "cuttlefish-runtime" - pod, err := New("dev").RenderPod(context.Background(), testExporterSet(), &virtualtargetv1alpha1.VirtualTargetClass{}, params, nil, nil) - if err != nil { - t.Fatal(err) - } - return pod -} - func TestRenderPod_privateImageCopy(t *testing.T) { - for _, readOnly := range []bool{true, false} { - pod := renderTestPod(t, map[string]interface{}{"image_volume_claim": "images", "image_volume_read_only": readOnly}) - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim != nil && !volume.PersistentVolumeClaim.ReadOnly { - t.Fatal("source claim is writable") - } - if volume.Name == "cvd-images" && volume.EmptyDir == nil { - t.Fatal("missing private image volume") - } + pod := renderTestPod(t, map[string]interface{}{"image_volume_claim": "images"}) + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && !volume.PersistentVolumeClaim.ReadOnly { + t.Fatal("source claim is writable") } - copy := pod.Spec.InitContainers[0] - if copy.Name != "copy-images" || !copy.VolumeMounts[0].ReadOnly || copy.VolumeMounts[1].ReadOnly { - t.Fatal("invalid copy mounts") + if volume.Name == "cvd-images" && volume.EmptyDir == nil { + t.Fatal("missing private image volume") } - for _, container := range pod.Spec.InitContainers[1:] { - for _, mount := range container.VolumeMounts { - if mount.Name == "image-source" { - t.Fatalf("%s can access shared source", container.Name) - } + } + copy := pod.Spec.InitContainers[0] + if copy.Name != "copy-images" || !copy.VolumeMounts[0].ReadOnly || copy.VolumeMounts[1].ReadOnly { + t.Fatal("invalid copy mounts") + } + for _, container := range pod.Spec.InitContainers[1:] { + for _, mount := range container.VolumeMounts { + if mount.Name == "image-source" { + t.Fatalf("%s can access shared source", container.Name) } } } } -func TestRenderPod_healthAndRelayIsolation(t *testing.T) { +func TestRenderPod_healthGate(t *testing.T) { pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) gate := pod.Spec.InitContainers[len(pod.Spec.InitContainers)-1] - if gate.Name != "wait-for-cuttlefish" || !strings.Contains(gate.Command[2], "127.0.0.1:2081/_debug/statusz") { - t.Fatal("missing API startup gate") + if gate.Name != gateContainerName || gate.Command[3] != "--wait" || gate.Command[4] != hostOrchestratorURL { + t.Fatalf("missing API startup gate: %#v", gate.Command) + } + if gate.Image != pod.Spec.Containers[0].Image { + t.Fatal("gate must run in the exporter image") } probe := pod.Spec.Containers[0].LivenessProbe - if probe == nil || probe.Exec.Command[2] != "jumpstarter_driver_cuttlefish.health" { + if probe == nil || probe.Exec.Command[2] != "jumpstarter_driver_cuttlefish.health" || probe.Exec.Command[3] != healthStatePath { t.Fatal("missing runtime failure detection") } - for _, container := range pod.Spec.InitContainers { - if container.Name == "cuttlefish-relay" && strings.Count(container.Command[2], "bind=127.0.0.1") != 2 { - t.Fatal("relay exposed outside Pod") - } + if !hasMount(pod.Spec.Containers[0].VolumeMounts, "cvd-state", runtimeIDMount) { + t.Fatal("exporter cannot read the runtime marker") } } @@ -145,24 +122,29 @@ func TestRenderPod_storageBudgets(t *testing.T) { total.Add(*volume.EmptyDir.SizeLimit) } } - for _, container := range pod.Spec.InitContainers { - if container.Name == "fetch-images" || container.Name == "cuttlefish" { - request := container.Resources.Requests[corev1.ResourceEphemeralStorage] - limit := container.Resources.Limits[corev1.ResourceEphemeralStorage] - if request.Cmp(total) != 0 || limit.Cmp(total) != 0 { - t.Fatalf("%s storage does not cover volumes: %v", container.Name, container.Resources) - } + if total.Cmp(resource.MustParse("15Gi")) != 0 { + t.Fatalf("volume total = %s", total.String()) + } + for _, name := range []string{"fetch-images", runtimeContainerName} { + container := initContainer(t, pod, name) + request := container.Resources.Requests[corev1.ResourceEphemeralStorage] + limit := container.Resources.Limits[corev1.ResourceEphemeralStorage] + if request.Cmp(total) != 0 || limit.Cmp(total) != 0 { + t.Fatalf("%s storage does not cover volumes: %v", name, container.Resources) } } } func TestStorageValidation(t *testing.T) { for _, value := range []interface{}{"", "0", "-1Gi", "invalid", 42} { - _, _, _, err := storageSizes(map[string]interface{}{"storage": map[string]interface{}{"imageSize": value}}) + _, err := resolveStorageConfig(map[string]interface{}{"fetch_images": true, "storage": map[string]interface{}{"imageSize": value}}) if err == nil { t.Fatalf("accepted invalid size %v", value) } } + if _, err := resolveStorageConfig(map[string]interface{}{"fetch_images": true, "storage": "invalid"}); err == nil { + t.Fatal("accepted non-object storage") + } budget := resource.MustParse("10Gi") for _, resources := range []corev1.ResourceRequirements{ {Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, @@ -190,12 +172,14 @@ func TestEnrichExporterExport(t *testing.T) { } cuttlefish := configFor(t, result[0]) - if cuttlefish["host"] != "127.0.0.1" || cuttlefish["port"] != float64(hostOrchestratorPort) { + if cuttlefish["host"] != "127.0.0.1" || cuttlefish["port"] != float64(hostOrchestratorPort) || cuttlefish["scheme"] != "http" { t.Errorf("Cuttlefish endpoint = %#v", cuttlefish) } + if fmt.Sprint(cuttlefish["health_ports"]) != fmt.Sprint([]interface{}{float64(netsimPort), float64(hciPort)}) { + t.Errorf("health_ports = %v", cuttlefish["health_ports"]) + } envConfig := cuttlefish["env_config"].(map[string]interface{}) - instances := envConfig["instances"].([]interface{}) - instance := instances[0].(map[string]interface{}) + instance := envConfig["instances"].([]interface{})[0].(map[string]interface{}) graphics := instance["graphics"].(map[string]interface{}) if graphics["gpu_mode"] != "none" { t.Errorf("gpu_mode = %v", graphics["gpu_mode"]) @@ -206,14 +190,11 @@ func TestEnrichExporterExport(t *testing.T) { } netsim := configFor(t, result[1]) - if netsim["host"] != "127.0.0.1" || netsim["port"] != float64(netsimRelayPort) { + if netsim["host"] != "127.0.0.1" || netsim["port"] != float64(netsimPort) { t.Errorf("netsim config = %#v", netsim) } - if _, exists := netsim["transport"]; exists { - t.Error("netsim driver does not accept transport") - } btPeer := configFor(t, result[2]) - if btPeer["transport"] != fmt.Sprintf("tcp-client:127.0.0.1:%d", hciRelayPort) { + if btPeer["transport"] != fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort) { t.Errorf("bt_peer config = %#v", btPeer) } } @@ -225,7 +206,6 @@ func TestEnrichExporterExportDefaultsPodSafeGraphicsAndVM(t *testing.T) { if err != nil { t.Fatal(err) } - config := configFor(t, result[0]) envConfig := config["env_config"].(map[string]interface{}) instance := envConfig["instances"].([]interface{})[0].(map[string]interface{}) @@ -242,55 +222,39 @@ func TestEnrichExporterExportRejectsExternalEndpoints(t *testing.T) { t.Errorf("accepted external endpoint %v", config) } } -} - -func configFor(t *testing.T, driver virtualtargetv1alpha1.DriverConfig) map[string]interface{} { - t.Helper() - var config map[string]interface{} - if err := json.Unmarshal(driver.Config.Raw, &config); err != nil { + // Restating the pinned values is fine, including as JSON numbers. + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(map[string]interface{}{ + "host": "127.0.0.1", "port": 2081.0, "instance_num": 1, "scheme": "http", + })} + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{driver}, nil); err != nil { t.Fatal(err) } - return config } -func hasVolume(volumes []corev1.Volume, name, path string) bool { - for _, volume := range volumes { - if volume.Name == name && volume.HostPath != nil && volume.HostPath.Path == path { - return true +func TestEnrichExporterExportPinsSimulatorEndpoints(t *testing.T) { + cuttlefish := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType} + for _, driver := range []virtualtargetv1alpha1.DriverConfig{ + {Name: "netsim", Type: netsimDriverType, Config: mustJSON(map[string]interface{}{"host": "netsim.example.com"})}, + {Name: "netsim", Type: netsimDriverType, Config: mustJSON(map[string]interface{}{"port": 9999})}, + {Name: "bt_peer", Type: btPeerDriverType, Config: mustJSON(map[string]interface{}{"transport": "tcp-client:10.0.0.1:7300"})}, + } { + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{cuttlefish, driver}, nil); err == nil { + t.Errorf("accepted external %s endpoint", driver.Name) } } - return false -} - -func mustJSON(value interface{}) *apiextensionsv1.JSON { - raw, err := json.Marshal(value) - if err != nil { - panic(err) - } - return &apiextensionsv1.JSON{Raw: raw} -} - -func testExporterSet() *virtualtargetv1alpha1.ExporterSet { - return &virtualtargetv1alpha1.ExporterSet{ - ObjectMeta: metav1.ObjectMeta{Name: "cuttlefish", Namespace: "default", UID: "test-uid"}, - Spec: virtualtargetv1alpha1.ExporterSetSpec{Template: virtualtargetv1alpha1.ExporterSetTemplate{Spec: virtualtargetv1alpha1.ExporterTemplateSpec{Drivers: []virtualtargetv1alpha1.DriverConfig{{Name: "cuttlefish", Type: cuttlefishDriverType}}}}}, + // Restating the in-Pod endpoints, and unrelated keys, stay accepted. + drivers := []virtualtargetv1alpha1.DriverConfig{ + cuttlefish, + {Name: "netsim", Type: netsimDriverType, Config: mustJSON(map[string]interface{}{"host": "127.0.0.1", "port": 7681.0})}, + {Name: "bt_peer", Type: btPeerDriverType, Config: mustJSON(map[string]interface{}{"address": "00:11:22:33:44:55"})}, } -} - -func TestRelayPortValidation(t *testing.T) { - for _, params := range []map[string]interface{}{ - {"hci_relay_port": 17681}, {"netsim_relay_port": 7681}, {"hci_relay_port": 7300}, - {"netsim_relay_port": 0}, {"netsim_relay_port": -1}, {"netsim_relay_port": 65536}, - {"netsim_relay_port": 1234.5}, {"netsim_relay_port": "1234"}, {"netsim_relay_port": true}, - {"netsim_relay_port": 80}, {"netsim_relay_port": 19531}, {"hci_relay_port": 6521}, - {"host_orchestrator_port": 9999}, {"netsim_relay_port": 2081}, {"netsim_relay_port": 15550}, - } { - if _, _, err := relayPorts(params); err == nil { - t.Errorf("accepted %v", params) - } + result, err := New("dev").EnrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) } - if n, h, err := relayPorts(map[string]interface{}{"netsim_relay_port": float64(27681), "hci_relay_port": 27300}); err != nil || n != 27681 || h != 27300 { - t.Fatalf("valid ports: %d %d %v", n, h, err) + if btPeer := configFor(t, result[2]); btPeer["address"] != "00:11:22:33:44:55" || + btPeer["transport"] != fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort) { + t.Errorf("bt_peer config = %#v", btPeer) } } @@ -320,20 +284,36 @@ func TestManagedContract(t *testing.T) { if _, err := New("dev").EnrichExporterExport(append(drivers, drivers[0]), nil); err == nil { t.Fatal("accepted multiple Cuttlefish drivers") } - if _, err := New("dev").EnrichExporterExport(drivers, map[string]interface{}{"vm_memory_mb": 12.5}); err == nil { - t.Fatal("accepted fractional VM memory") + for _, params := range []map[string]interface{}{{"vm_memory_mb": 12.5}, {"vm_cpus": "4"}, {"vm_cpus": 0}} { + if _, err := New("dev").EnrichExporterExport(drivers, params); err == nil { + t.Fatalf("accepted guest parameters %v", params) + } } enriched, err := New("dev").EnrichExporterExport(drivers, nil) if err != nil { t.Fatal(err) } config := configFor(t, enriched[0]) - vm := config["env_config"].(map[string]interface{})["instances"].([]interface{})[0].(map[string]interface{})["vm"].(map[string]interface{}) - if config["managed"] != true || vm["crosvm"].(map[string]interface{})["vhost_user_vsock"] != "true" { + envConfig := config["env_config"].(map[string]interface{}) + vm := envConfig["instances"].([]interface{})[0].(map[string]interface{})["vm"].(map[string]interface{}) + if config["managed"] != true || envConfig["netsim_bt"] != true || vm["crosvm"].(map[string]interface{})["vhost_user_vsock"] != "true" { t.Fatalf("missing managed isolation: %v", config) } } +func TestGuestSpecPrefersTemplateValues(t *testing.T) { + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(map[string]interface{}{ + "env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"cpus": 2}}}}, + })} + _, guest, err := enrichDrivers([]virtualtargetv1alpha1.DriverConfig{driver}, map[string]interface{}{"vm_cpus": 8, "vm_memory_mb": 4096}) + if err != nil { + t.Fatal(err) + } + if guest != (guestSpec{cpus: 2, memoryMB: 4096}) { + t.Fatalf("guest = %+v", guest) + } +} + func TestRuntimeMemory(t *testing.T) { for _, tc := range []struct { name, memory, request, limit, want string @@ -369,10 +349,9 @@ func TestRuntimeMemory(t *testing.T) { if err != nil { t.Fatal(err) } - for _, c := range pod.Spec.InitContainers { - if c.Name == "cuttlefish" && c.Resources.Requests.Memory().Cmp(resource.MustParse(tc.want)) != 0 { - t.Fatalf("memory = %s, want %s", c.Resources.Requests.Memory(), tc.want) - } + runtime := initContainer(t, pod, runtimeContainerName) + if runtime.Resources.Requests.Memory().Cmp(resource.MustParse(tc.want)) != 0 { + t.Fatalf("memory = %s, want %s", runtime.Resources.Requests.Memory(), tc.want) } }) } @@ -395,6 +374,7 @@ func TestPodIsolation(t *testing.T) { } for _, params := range []map[string]interface{}{ {"fetch_images": true, "service_account_name": "cuttlefish-runtime", "runtime_privileged": false}, + {"fetch_images": true, "service_account_name": "cuttlefish-runtime"}, {"fetch_images": true, "runtime_privileged": true}, {"fetch_images": true, "runtime_privileged": true, "service_account_name": "default"}, {"fetch_images": true, "runtime_privileged": true, "service_account_name": "Invalid_Name"}, @@ -403,32 +383,83 @@ func TestPodIsolation(t *testing.T) { t.Fatalf("accepted %v", params) } } + es.Spec.RecycleStrategy = virtualtargetv1alpha1.RecycleStrategyInPlaceReuse + if _, err := New("dev").RenderPod(context.Background(), es, &virtualtargetv1alpha1.VirtualTargetClass{}, map[string]interface{}{ + "fetch_images": true, "runtime_privileged": true, "service_account_name": "cuttlefish-runtime", + }, nil, nil); err == nil { + t.Fatal("accepted InPlaceReuse") + } } -func TestRelaySupervisorExitsWhenEitherRelayFails(t *testing.T) { - pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) - var command []string - for _, c := range pod.Spec.InitContainers { - if c.Name == "cuttlefish-relay" { - command = c.Command +func renderTestPod(t *testing.T, params map[string]interface{}) *corev1.Pod { + t.Helper() + params["runtime_privileged"] = true + params["service_account_name"] = "cuttlefish-runtime" + pod, err := New("dev").RenderPod(context.Background(), testExporterSet(), &virtualtargetv1alpha1.VirtualTargetClass{}, params, nil, nil) + if err != nil { + t.Fatal(err) + } + return pod +} + +func initContainer(t *testing.T, pod *corev1.Pod, name string) corev1.Container { + t.Helper() + for _, container := range pod.Spec.InitContainers { + if container.Name == name { + return container } } - for _, failingPort := range []int{netsimRelayPort, hciRelayPort} { - t.Run(fmt.Sprint(failingPort), func(t *testing.T) { - dir := t.TempDir() - script := fmt.Sprintf("#!/bin/sh\ncase \"$1\" in TCP-LISTEN:%d,*) exit 1;; esac\nexec sleep 30\n", failingPort) - if err := os.WriteFile(filepath.Join(dir, "socat"), []byte(script), 0755); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, command[0], command[1:]...) - cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) - cmd.WaitDelay = time.Second - err := cmd.Run() - if err == nil || ctx.Err() != nil { - t.Fatalf("supervisor did not promptly fail: %v, %v", err, ctx.Err()) - } - }) + t.Fatalf("init container %q missing", name) + return corev1.Container{} +} + +func configFor(t *testing.T, driver virtualtargetv1alpha1.DriverConfig) map[string]interface{} { + t.Helper() + var config map[string]interface{} + if err := json.Unmarshal(driver.Config.Raw, &config); err != nil { + t.Fatal(err) + } + return config +} + +func hasVolume(volumes []corev1.Volume, name, path string) bool { + for _, volume := range volumes { + if volume.Name == name && volume.HostPath != nil && volume.HostPath.Path == path { + return true + } + } + return false +} + +func hasMount(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} + +func hasEnv(env []corev1.EnvVar, name, value string) bool { + for _, variable := range env { + if variable.Name == name && variable.Value == value { + return true + } + } + return false +} + +func mustJSON(value interface{}) *apiextensionsv1.JSON { + raw, err := json.Marshal(value) + if err != nil { + panic(err) + } + return &apiextensionsv1.JSON{Raw: raw} +} + +func testExporterSet() *virtualtargetv1alpha1.ExporterSet { + return &virtualtargetv1alpha1.ExporterSet{ + ObjectMeta: metav1.ObjectMeta{Name: "cuttlefish", Namespace: "default", UID: "test-uid"}, + Spec: virtualtargetv1alpha1.ExporterSetSpec{Template: virtualtargetv1alpha1.ExporterSetTemplate{Spec: virtualtargetv1alpha1.ExporterTemplateSpec{Drivers: []virtualtargetv1alpha1.DriverConfig{{Name: "cuttlefish", Type: cuttlefishDriverType}}}}}, } } diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py index 324347d2e..d8112e080 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py @@ -39,7 +39,18 @@ def check(state_path: str) -> None: if cvds[0].get("status") != "Running": raise RuntimeError("CVD stopped unexpectedly") if not set(state["ports"]).issubset(listening_ports()): - raise RuntimeError("Cuttlefish simulator or relay listener is missing") + raise RuntimeError("Cuttlefish simulator listener is missing") + + +def wait_ready(url: str, attempts: int = 60, interval: float = 5) -> None: + """Startup gate: block until Host Orchestrator answers, so the exporter never registers early.""" + for _ in range(attempts): + try: + with urllib.request.urlopen(f"{url}/_debug/statusz", timeout=3): + return + except Exception: + time.sleep(interval) + raise RuntimeError(f"Host Orchestrator at {url} did not become ready") def listening_ports() -> set[int]: @@ -59,6 +70,8 @@ def listening_ports() -> set[int]: if sys.argv[1] == "--run-exporter": initialize(sys.argv[2], sys.argv[3], sys.argv[4]) os.execvp("jmp", ["jmp", "run", "--exporter-config", sys.argv[5]]) + elif sys.argv[1] == "--wait": + wait_ready(sys.argv[2]) else: check(sys.argv[1]) except Exception as exc: diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py index 969f6a6ab..a5a55a9a3 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py @@ -16,7 +16,7 @@ def health_state(tmp_path): return tmp_path / "health.json", { "runtime_id_path": str(runtime_id), "runtime_id": "runtime-1", "url": "http://127.0.0.1:2081", "state": "running", - "group": "cvd", "name": "1", "ports": [7681, 7300, 17681, 17300], + "group": "cvd", "name": "1", "ports": [7681, 7300], } @@ -44,8 +44,8 @@ def test_guest_failure(health_state, cvds): run_check(health_state, cvds=cvds) -@pytest.mark.parametrize("missing", [7681, 7300, 17681, 17300]) -def test_simulator_or_relay_failure(health_state, missing): +@pytest.mark.parametrize("missing", [7681, 7300]) +def test_simulator_listener_failure(health_state, missing): ports = set(health_state[1]["ports"]) - {missing} with pytest.raises(RuntimeError, match="listener is missing"): run_check(health_state, ports=ports) @@ -97,3 +97,16 @@ def test_warm_exporter_before_first_lease(tmp_path): runtime_id.write_text("runtime-2") with pytest.raises(RuntimeError, match="runtime restarted"): check(str(state_path)) + + +def test_wait_ready_gate(): + from .health import wait_ready + + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", return_value=io.BytesIO()) as urlopen: + wait_ready("http://127.0.0.1:2081", attempts=1, interval=0) + assert urlopen.call_args.args[0] == "http://127.0.0.1:2081/_debug/statusz" + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", side_effect=OSError("refused")), \ + patch("jumpstarter_driver_cuttlefish.health.time.sleep") as sleep: + with pytest.raises(RuntimeError, match="did not become ready"): + wait_ready("http://127.0.0.1:2081", attempts=3, interval=5) + assert sleep.call_count == 3