diff --git a/controller/cmd/exporter-set-controller/main.go b/controller/cmd/exporter-set-controller/main.go index 93eb6a9ef..6bcf56d69 100644 --- a/controller/cmd/exporter-set-controller/main.go +++ b/controller/cmd/exporter-set-controller/main.go @@ -36,6 +36,7 @@ import ( jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset" + "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish" "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/qemu" ) @@ -159,9 +160,14 @@ func main() { // Add new provisioners here as they are implemented. func selectProvisioner(name string) (exporterset.Provisioner, error) { switch name { + case cuttlefish.ProvisionerName: + return cuttlefish.New(version), nil case qemu.ProvisionerName: return qemu.New(version), nil default: - return nil, fmt.Errorf("unknown provisioner %q; supported: %s", name, qemu.ProvisionerName) + return nil, fmt.Errorf( + "unknown provisioner %q; supported: %s, %s", + name, qemu.ProvisionerName, cuttlefish.ProvisionerName, + ) } } diff --git a/controller/deploy/operator/config/rbac/role.yaml b/controller/deploy/operator/config/rbac/role.yaml index 6bc4f8ec7..e6a4cec31 100644 --- a/controller/deploy/operator/config/rbac/role.yaml +++ b/controller/deploy/operator/config/rbac/role.yaml @@ -171,6 +171,17 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - get + - list + - patch + - update + - watch - apiGroups: - operator.jumpstarter.dev resources: diff --git a/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go b/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go index 46a842cc3..4e2501663 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/exporterset.go @@ -517,6 +517,11 @@ func exporterSetPolicyRules() []rbacv1.PolicyRule { Resources: []string{"leases"}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, + }, { APIGroups: []string{""}, Resources: []string{"pods"}, diff --git a/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go b/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go index e58351745..e3dc39d4d 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go @@ -120,6 +120,16 @@ var _ = Describe("exporterSetPolicyRules", func() { Expect(groups).To(HaveKey("coordination.k8s.io")) }) + It("should reconcile runtime network isolation policies", func() { + for _, rule := range rules { + if containsString(rule.APIGroups, "networking.k8s.io") && containsString(rule.Resources, "networkpolicies") { + Expect(rule.Verbs).To(ContainElements("get", "list", "watch", "create", "update", "patch")) + return + } + } + Fail("no rule found for runtime network policies") + }) + It("should grant read-only access on exportersets (no create/update/delete)", func() { for _, rule := range rules { if containsString(rule.APIGroups, "virtualtarget.jumpstarter.dev") && diff --git a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 1a15ddf47..3e7706e9d 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -90,6 +90,7 @@ type JumpstarterReconciler struct { // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete // Networking resources +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete diff --git a/controller/internal/controller/lease_controller_test.go b/controller/internal/controller/lease_controller_test.go index 26e3d6200..e207c61e5 100755 --- a/controller/internal/controller/lease_controller_test.go +++ b/controller/internal/controller/lease_controller_test.go @@ -1492,7 +1492,7 @@ var _ = Describe("Scheduled Leases", func() { When("creating lease with BeginTime + Duration (scheduled lease)", func() { It("should wait until BeginTime before acquiring exporter", func() { lease := leaseDutA2Sec.DeepCopy() - futureTime := metav1.NewTime(time.Now().Truncate(time.Second).Add(1 * time.Second)) + futureTime := metav1.NewTime(time.Now().Add(2 * time.Second).Truncate(time.Second)) lease.Spec.BeginTime = &futureTime lease.Spec.Duration = &metav1.Duration{Duration: 1 * time.Second} lease.Spec.EndTime = nil diff --git a/controller/internal/exporterset/networkpolicy_test.go b/controller/internal/exporterset/networkpolicy_test.go new file mode 100644 index 000000000..102dbf2ca --- /dev/null +++ b/controller/internal/exporterset/networkpolicy_test.go @@ -0,0 +1,94 @@ +package exporterset + +import ( + "context" + "errors" + "strings" + "testing" + + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestCuttlefishNetworkPolicyReconciliation(t *testing.T) { + scheme := newScheme(t) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + es := makeExporterSet() + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &ExporterSetReconciler{Client: c, Scheme: scheme, Provisioner: cuttlefish.New("dev")} + ctx := context.Background() + if err := r.syncNetworkPolicy(ctx, es); err != nil { + t.Fatal(err) + } + policy := &networkingv1.NetworkPolicy{} + key := client.ObjectKey{Namespace: es.Namespace, Name: "cuttlefish-" + string(es.UID)} + if err := c.Get(ctx, key, policy); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(policy, es) || len(policy.Spec.Ingress) != 0 { + t.Fatalf("unowned or permissive policy: %#v", policy) + } + policy.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{{}} + if err := c.Update(ctx, policy); err != nil { + t.Fatal(err) + } + if err := r.syncNetworkPolicy(ctx, es); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, key, policy); err != nil { + t.Fatal(err) + } + if len(policy.Spec.Ingress) != 0 { + t.Fatal("policy drift not corrected") + } + if err := c.Delete(ctx, policy); err != nil { + t.Fatal(err) + } + if err := r.syncNetworkPolicy(ctx, es); err != nil { + t.Fatal(err) + } + if err := c.Get(ctx, key, policy); err != nil { + t.Fatal("deleted policy not recreated", err) + } +} + +func TestPolicyFailurePreventsWorkloadCreation(t *testing.T) { + scheme := newScheme(t) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + es := makeExporterSet() + vtc := &virtualtargetv1alpha1.VirtualTargetClass{ + ObjectMeta: metav1.ObjectMeta{Name: es.Spec.VirtualTargetClassName, Namespace: es.Namespace}, + Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Provisioner: cuttlefish.ProvisionerName}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(es, vtc).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*networkingv1.NetworkPolicy); ok { + return errors.New("network policy denied") + } + return c.Create(ctx, obj, opts...) + }, + }).Build() + r := &ExporterSetReconciler{Client: c, Scheme: scheme, Provisioner: cuttlefish.New("dev")} + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(es)}) + if err == nil || !strings.Contains(err.Error(), "network policy denied") { + t.Fatalf("expected policy error: %v", err) + } + var pods corev1.PodList + if err := c.List(context.Background(), &pods); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatal("created workload without network isolation") + } +} diff --git a/controller/internal/exporterset/provisioners/cuttlefish/README.md b/controller/internal/exporterset/provisioners/cuttlefish/README.md new file mode 100644 index 000000000..a54c9c42e --- /dev/null +++ b/controller/internal/exporterset/provisioners/cuttlefish/README.md @@ -0,0 +1,251 @@ +# Cuttlefish ExporterSets + +Each exporter owns one CVD. The managed backend uses Host Orchestrator over +HTTP inside the Pod, crosvm with private userspace VSOCK, and netsim Bluetooth. +The standalone Python driver continues to support externally managed HTTP hosts. +An exec backend is a separate follow-up. + +## Workload admission and networking + +Create a dedicated service account in the ExporterSet namespace and set +`parameters.service_account_name` to its name. `default` is rejected. The Pod +does not mount a Kubernetes API token. `runtime_privileged: true` is required; +`false` is rejected until device permissions and capabilities are supported. + +On Kubernetes, create the namespace and workload account. If Pod Security +Admission is enabled, the namespace must permit privileged workloads; the +`baseline` and `restricted` profiles reject this Pod. For a dedicated test +namespace: + +```sh +kubectl create namespace cuttlefish-lab +kubectl label namespace cuttlefish-lab pod-security.kubernetes.io/enforce=privileged +kubectl -n cuttlefish-lab create serviceaccount cuttlefish-runtime +``` + +This namespace setting permits privileged workloads for every account in that +namespace, so restrict who can create workloads there. Other admission policies +must also allow the Pod's privileged containers, hostPath devices and container +UIDs. The workload account needs no Kubernetes API permissions. Nodes must expose +the required KVM and networking devices; VM-based nodes need nested virtualization. +The cluster must support native sidecar containers: the runtime init containers +use `restartPolicy: Always`. +See [Kubernetes Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). + +On OpenShift, a cluster administrator must grant that workload account access +to an SCC permitting privileged containers, hostPath devices, and the UIDs used +by **all** containers, including image copy/fetch and permission init containers. +For initial testing, a scoped grant to the built-in privileged SCC is: + +```sh +oc -n cuttlefish-lab create serviceaccount cuttlefish-runtime +oc adm policy add-scc-to-user privileged -z cuttlefish-runtime -n cuttlefish-lab +``` + +This grant is for the workload account, not the ExporterSet controller. Merely +setting `runtime_privileged` cannot grant SCC admission. Check the admitted Pod's +`openshift.io/scc` annotation, completion of init containers, and an actual guest +boot. A server-side dry run only establishes admission, not device access. +See [OpenShift SCC documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/authentication_and_authorization/managing-pod-security-policies). + +The controller creates and reconciles an ExporterSet-owned NetworkPolicy before +creating workloads. It denies Pod ingress, including Host Orchestrator, nginx, +ADB and simulator listeners, while leaving outbound exporter connections and +same-Pod loopback traffic available. Deploy with a CNI that enforces NetworkPolicy. +Other policies must not grant ingress to these Pods: Kubernetes allow rules are +additive. Verify denial from a second Pod, using the actual runtime image; +loopback addresses in driver configuration do not change the server's listeners. +NetworkPolicy does not isolate privileged containers from their node. + +Controllers deployed outside the operator also need `get,list,watch,create,update,patch` +on `networking.k8s.io/networkpolicies`. The operator supplies these permissions. +Existing Pods must be drained and replaced to receive the isolation label, +service account, runtime marker and managed driver configuration. + +## Configuration and resource allocation + +The provisioner requires exactly one Cuttlefish driver and one +`env_config.instances` entry. It pins the managed endpoint to +`http://127.0.0.1:2081` and `instance_num` to 1; a template that sets different +values is rejected. The upstream image fixes Host Orchestrator to 2081 behind +nginx on 2080, and all containers share the Pod network namespace, so the netsim +and bt_peer drivers are pointed directly at the simulators on loopback ports +7681 and 7300. + +Guest defaults are 4 CPUs and 8192 MiB. Runtime memory requests default to the +**effective** guest memory plus `runtime_memory_overhead_mb` (2048 MiB by default). +Explicit requests and limits must cover that budget. Increase the overhead for +larger simulator workloads. Guest values in driver `env_config` take precedence +over `vm_cpus` and `vm_memory_mb` when calculating the budget. CPU requests default +to guest CPUs, or to an explicit CPU limit; CPU overcommit remains configurable. + +`create_cvd()` in managed mode accepts only the exact configured `env_config`, +checks the entire Host Orchestrator inventory, and serializes creation with other +lifecycle operations. Destroy the existing CVD before creating another. This +prevents alternate API payloads or concurrent calls from exceeding the configured +instance count and memory budget. Arbitrary template code and driver imports are +administrator-controlled; they are not a security boundary against a malicious +cluster administrator. + +## VSOCK and Bluetooth compatibility + +Managed configuration sets: + +```yaml +env_config: + netsim_bt: true + instances: + - vm: + crosvm: + vhost_user_vsock: "true" +``` + +The string value is required by the upstream configuration schema. The runtime +and guest must support that backend. The provisioner does not mount +`/dev/vhost-vsock`; each Pod has private runtime files and Unix sockets. A +privileged runtime still has broad node access, so this is not containment of a +compromised runtime. QEMU, gem5, disabled userspace VSOCK, and standalone RootCanal +(`netsim_bt: false`) are rejected for this managed backend. The referenced +upstream standalone RootCanal proxy does not propagate the userspace VSOCK flag. + +Before approving a runtime/build pair, boot two Pods on the same node using the +default identical guest CID, verify their generated configuration and private +`vhost.socket`/`vm.vsock` paths, and exercise netsim plus Bluetooth peer traffic +in both leases. Stop one guest and verify the other remains usable. Source/unit +tests cannot establish compatibility of a mutable image tag or guest build. + +## Image PVCs and reproducibility + +Exactly one image source is required: set either `fetch_images: true` or +`image_volume_claim`. Setting both, or neither, is rejected. Fetching downloads +`default_build` into each Pod; a claim provisions from a prewarmed PVC: + +```yaml +# Fetch per Pod. +parameters: + fetch_images: true + default_build: "/aosp_cf_x86_64_auto-userdebug" +``` + +```yaml +# Prewarmed PVC. default_build is unused; the images come from the claim. +parameters: + image_volume_claim: cuttlefish-images +``` + +A prewarmed image PVC is mounted read-only, then copied into a private writable +`emptyDir`. It remains a Pod volume after the init container exits. Read-only +mounting does **not** change the PVC's access mode or remove attachment constraints. + +For a pool spanning nodes, use storage supporting ReadOnlyMany or ReadWriteMany +with the required topology. For ReadWriteOnce, keep readers on one compatible +node; ReadWriteOncePod permits only one Pod. Alternatively fetch images per Pod. +See [Kubernetes access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). + +Use an immutable runtime manifest digest and Android build ID for +repeatable replacements. This example is a template: substitute verified values +from a tested pair; the placeholders are not a published compatibility claim. +Pin the exporter image built with this provisioner/driver change too. + +```yaml +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: VirtualTargetClass +metadata: + name: cuttlefish + namespace: cuttlefish-lab +spec: + provisioner: cuttlefish.jumpstarter.dev + parameters: + service_account_name: cuttlefish-runtime + runtime_privileged: true + fetch_images: true + default_build: "/aosp_cf_x86_64_auto-userdebug" + vm_cpus: 4 + vm_memory_mb: 8192 + runtime_memory_overhead_mb: 2048 + images: + runtime: + image: "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orchestration/cuttlefish-orchestration@sha256:" + exporter: + image: "quay.io/jumpstarter-dev/jumpstarter@sha256:" + scheduling: + resources: + requests: + cpu: "4" + memory: 10Gi + limits: + memory: 10Gi +``` + +The class carries the provisioner parameters and images; the drivers live in the +ExporterSet template. Use `ExitAndReplace`, and keep `env_config.instances` to a +single entry. The provisioner fills in the managed endpoints, so the driver +configuration below only needs what it overrides: + +```yaml +apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 +kind: ExporterSet +metadata: + name: cuttlefish + namespace: cuttlefish-lab +spec: + minReplicas: 0 + maxReplicas: 4 + minAvailableReplicas: 1 + recycleStrategy: ExitAndReplace + virtualTargetClassName: cuttlefish + selector: + matchLabels: + device: cuttlefish + template: + metadata: + labels: + device: cuttlefish + spec: + drivers: + - name: cuttlefish + type: jumpstarter_driver_cuttlefish.driver.Cuttlefish + config: + env_config: + instances: + - vm: + cpus: 4 + memory_mb: 8192 + - name: netsim + type: jumpstarter_driver_netsim.driver.Netsim + - name: bt_peer + type: jumpstarter_driver_bt_peer.driver.BtPeer +``` + +Record both the resolved runtime image digest and the guest `fetcher_config.json` +with validation results. A prewarmed PVC needs the same build provenance. + +## Failure and recovery behavior + +The exporter liveness probe reads state written atomically by the managed driver. +It always checks Host Orchestrator availability and a per-start runtime ID. +When the guest is expected to run, it also checks the CVD inventory/status and +simulator TCP listeners in the shared network namespace. It inspects +listeners instead of opening HCI connections that could disturb Bluetooth peers. +A listening socket alone does not prove simulator protocol correctness. + +Intentional stop, destroy and reset leave the Pod healthy without a guest. +Create/start/restart/powerwash receive bounded transition time; failed or expired +operations fail health checks. A runtime sidecar restart invalidates the exporter +even if its HTTP API comes back: it must not silently resume a lease after losing +runtime state. Six failed checks, ten seconds apart, terminate the exporter. + +With `ExitAndReplace`, the failed exporter remains associated with an active +lease. Release that lease to let the controller delete and replace the Pod; +reacquire a lease for a fresh device. Automatic replacement during an active +lease is not attempted. Use `ExitAndReplace` for managed Cuttlefish pools. + +Validate recovery on disposable leased Pods by terminating the VMM, netsim, and +then the runtime sidecar in separate trials. A component may recover +within the probe failure window; verify guest and simulator functionality after +recovery. For unrecovered failures and runtime sidecar restarts, confirm liveness +failure, client disconnect, retention while leased, and replacement after release. Also +exercise intentional power off/on and destroy/create, which must remain healthy. +Actual CNI enforcement and colocated VSOCK/BT trials are required integration +checks beyond the local regression tests. OpenShift deployments additionally +require SCC admission and guest boot validation. diff --git a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go new file mode 100644 index 000000000..ab41468d7 --- /dev/null +++ b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish.go @@ -0,0 +1,750 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cuttlefish implements the cuttlefish.jumpstarter.dev provisioner. +// Each exporter Pod owns one Cuttlefish runtime and one CVD. +package cuttlefish + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "maps" + "math" + "slices" + "strings" + + jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + ProvisionerName = "cuttlefish.jumpstarter.dev" + + DefaultExporterImage = "quay.io/jumpstarter-dev/jumpstarter:latest" + DefaultRuntimeImage = "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orchestration/cuttlefish-orchestration:stable" + + exporterConfigPath = "/etc/jumpstarter/exporters/config.yaml" + exporterNonRootUID int64 = 65532 + + cuttlefishDriverType = "jumpstarter_driver_cuttlefish.driver.Cuttlefish" + netsimDriverType = "jumpstarter_driver_netsim.driver.Netsim" + btPeerDriverType = "jumpstarter_driver_bt_peer.driver.BtPeer" + + // All containers share the Pod network namespace, so the exporter reaches + // Host Orchestrator and the simulators on loopback. The orchestration image + // reserves 2080 for nginx inside a Pod, so Host Orchestrator listens on 2081. + hostOrchestratorPort = 2081 + hostOrchestratorURL = "http://127.0.0.1:2081" + netsimPort = 7681 + hciPort = 7300 + + defaultGPUMode = "guest_swiftshader" + defaultVMCPUs = 4 + defaultVMMemoryMB = 8192 + defaultOverheadMB = 2048 + isolationLabel = "cuttlefish.jumpstarter.dev/exporter-set" + healthStatePath = "/tmp/jumpstarter-cuttlefish-health.json" + runtimeIDMount = "/run/cuttlefish-runtime" + runtimeIDPath = runtimeIDMount + "/runtime-id" + + fetchPath = "/home/vsoc-01/fetch" + cvdStatePath = "/var/tmp/cvd" + androidTmpPath = "/tmp/android" + imageSourcePath = "/image-source" + + runtimeContainerName = "cuttlefish" + gateContainerName = "wait-for-cuttlefish" +) + +// healthPorts are the simulator listeners the liveness probe expects while a guest runs. +var healthPorts = []int{netsimPort, hciPort} + +type Provisioner struct { + Version string +} + +type storageConfig struct { + imageClaim string + fetchImages bool + build string + imageSize, stateSize, tmpSize resource.Quantity + // budget is the ephemeral storage every container touching the volumes must reserve. + budget resource.Quantity +} + +// guestSpec is the effective guest size after template values override parameters. +type guestSpec struct { + cpus, memoryMB int +} + +type images struct { + exporter, runtime string + exporterPull, runtimePull corev1.PullPolicy +} + +func New(version string) *Provisioner { + return &Provisioner{Version: version} +} + +func (p *Provisioner) Name() string { + return ProvisionerName +} + +func (p *Provisioner) resolveImage(image string) string { + if p.Version == "" || p.Version == "dev" || strings.Contains(p.Version, "-g") { + return image + } + version := strings.TrimPrefix(p.Version, "v") + if base, ok := strings.CutSuffix(image, ":latest"); ok { + return base + ":" + version + } + return image +} + +func (p *Provisioner) resolveImageSpec(spec *virtualtargetv1alpha1.ImageSpec, defaultImage string) (string, corev1.PullPolicy) { + image := p.resolveImage(defaultImage) + pullPolicy := corev1.PullIfNotPresent + if spec != nil { + if spec.Image != "" { + image = spec.Image + } + if spec.ImagePullPolicy != "" { + pullPolicy = spec.ImagePullPolicy + } + } + return image, pullPolicy +} + +func (p *Provisioner) resolveImages(overrides *virtualtargetv1alpha1.ImageOverrides) images { + var exporterSpec, runtimeSpec *virtualtargetv1alpha1.ImageSpec + if overrides != nil { + exporterSpec, runtimeSpec = overrides.Exporter, overrides.Runtime + } + img := images{} + img.exporter, img.exporterPull = p.resolveImageSpec(exporterSpec, DefaultExporterImage) + img.runtime, img.runtimePull = p.resolveImageSpec(runtimeSpec, DefaultRuntimeImage) + return img +} + +func resolveStorageConfig(parameters map[string]interface{}) (storageConfig, error) { + config := storageConfig{ + imageSize: resource.MustParse("20Gi"), + stateSize: resource.MustParse("20Gi"), + tmpSize: resource.MustParse("4Gi"), + } + config.imageClaim, _ = parameters["image_volume_claim"].(string) + config.fetchImages, _ = parameters["fetch_images"].(bool) + if config.imageClaim != "" && config.fetchImages { + return config, fmt.Errorf("cuttlefish requires either image_volume_claim or fetch_images, not both") + } + if config.imageClaim == "" && !config.fetchImages { + return config, fmt.Errorf("cuttlefish requires image_volume_claim or fetch_images=true") + } + if config.fetchImages { + config.build = "aosp-android-latest-release/aosp_cf_x86_64_auto-userdebug" + if value, ok := parameters["default_build"].(string); ok && value != "" { + config.build = value + } + } + + if raw, exists := parameters["storage"]; exists { + storage, ok := raw.(map[string]interface{}) + if !ok { + return config, fmt.Errorf("parameters.storage must be an object") + } + for key, target := range map[string]*resource.Quantity{ + "imageSize": &config.imageSize, "stateSize": &config.stateSize, "tmpSize": &config.tmpSize, + } { + value, exists := storage[key] + if !exists { + continue + } + text, ok := value.(string) + quantity, err := resource.ParseQuantity(text) + if !ok || err != nil || quantity.Sign() <= 0 { + return config, fmt.Errorf("parameters.storage.%s must be a positive storage quantity", key) + } + *target = quantity + } + } + config.budget = config.imageSize.DeepCopy() + config.budget.Add(config.stateSize) + config.budget.Add(config.tmpSize) + config.budget.Add(resource.MustParse("1Gi")) // Container layers and logs need space beyond the volume budgets. + return config, nil +} + +func resolveServiceAccount(parameters map[string]interface{}) (string, error) { + if privileged, _ := parameters["runtime_privileged"].(bool); !privileged { + return "", fmt.Errorf("cuttlefish requires runtime_privileged=true; unprivileged device access is not supported") + } + name, _ := parameters["service_account_name"].(string) + if name == "" || name == "default" || len(validation.IsDNS1123Subdomain(name)) != 0 { + return "", fmt.Errorf("service_account_name must name a dedicated workload service account") + } + return name, nil +} + +func (p *Provisioner) RenderPod( + ctx context.Context, + exporterSet *virtualtargetv1alpha1.ExporterSet, + vtc *virtualtargetv1alpha1.VirtualTargetClass, + mergedParameters map[string]interface{}, + overrides *virtualtargetv1alpha1.ImageOverrides, + exporter *jumpstarterdevv1alpha1.Exporter, +) (*corev1.Pod, error) { + _ = ctx + if exporterSet.Spec.RecycleStrategy == virtualtargetv1alpha1.RecycleStrategyInPlaceReuse { + return nil, fmt.Errorf("managed Cuttlefish requires ExitAndReplace recycling") + } + storage, err := resolveStorageConfig(mergedParameters) + if err != nil { + return nil, err + } + serviceAccount, err := resolveServiceAccount(mergedParameters) + if err != nil { + return nil, err + } + // The reconciler persists the enriched drivers itself; rendering needs the + // validation and the effective guest size for the runtime budget. + _, guest, err := enrichDrivers(exporterSet.Spec.Template.Spec.Drivers, mergedParameters) + if err != nil { + return nil, err + } + resources, err := runtimeResources(vtc, guest, mergedParameters) + if err != nil { + return nil, err + } + if err := reserveStorage(&resources, storage.budget); err != nil { + return nil, err + } + img := p.resolveImages(overrides) + + pod := &corev1.Pod{ + ObjectMeta: podMeta(exporterSet, exporter), + Spec: corev1.PodSpec{ + // Never: ExitAndReplace relies on the exporter (main) exit completing the Pod. + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount, + AutomountServiceAccountToken: boolPtr(false), + InitContainers: initContainers(img, storage, resources), + Containers: []corev1.Container{exporterContainer(img)}, + Volumes: volumes(storage), + }, + } + if vtc.Spec.Scheduling != nil { + if vtc.Spec.Scheduling.NodeSelector != nil { + pod.Spec.NodeSelector = maps.Clone(vtc.Spec.Scheduling.NodeSelector) + } + if vtc.Spec.Scheduling.Tolerations != nil { + pod.Spec.Tolerations = append([]corev1.Toleration(nil), vtc.Spec.Scheduling.Tolerations...) + } + } + return pod, nil +} + +func podMeta(exporterSet *virtualtargetv1alpha1.ExporterSet, exporter *jumpstarterdevv1alpha1.Exporter) metav1.ObjectMeta { + meta := metav1.ObjectMeta{ + Namespace: exporterSet.Namespace, + Labels: maps.Clone(exporterSet.Spec.Template.Metadata.Labels), + Annotations: maps.Clone(exporterSet.Spec.Template.Metadata.Annotations), + } + if exporter != nil { + meta.Name = exporter.Name + } else { + meta.GenerateName = exporterSet.Name + "-" + } + if meta.Labels == nil { + meta.Labels = map[string]string{} + } + meta.Labels[isolationLabel] = string(exporterSet.UID) + return meta +} + +func exporterSecurityContext() *corev1.SecurityContext { + uid := exporterNonRootUID + return &corev1.SecurityContext{RunAsUser: &uid, RunAsNonRoot: boolPtr(true)} +} + +// exporterContainer runs jmp behind the health wrapper, which records the +// runtime marker before the exporter registers and backs the liveness probe. +func exporterContainer(img images) corev1.Container { + return corev1.Container{ + Name: "exporter", + Image: img.exporter, + ImagePullPolicy: img.exporterPull, + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", "--run-exporter", + healthStatePath, runtimeIDPath, hostOrchestratorURL, exporterConfigPath}, + Env: []corev1.EnvVar{{Name: "HOME", Value: "/tmp"}}, + SecurityContext: exporterSecurityContext(), + VolumeMounts: []corev1.VolumeMount{{Name: "cvd-state", MountPath: runtimeIDMount, ReadOnly: true}}, + // With restartPolicy Never, a failed liveness check ends the exporter and lets ExitAndReplace recycle the Pod. + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", healthStatePath}, + }}, + PeriodSeconds: 10, TimeoutSeconds: 10, FailureThreshold: 6, + }, + } +} + +// initContainers stages images, fixes ownership, starts the runtime as a +// native sidecar and gates the exporter on Host Orchestrator readiness. +func initContainers(img images, storage storageConfig, runtime corev1.ResourceRequirements) []corev1.Container { + stateMounts := []corev1.VolumeMount{ + {Name: "cvd-images", MountPath: fetchPath}, + {Name: "cvd-state", MountPath: cvdStatePath}, + {Name: "android-tmp", MountPath: androidTmpPath}, + } + deviceMounts := []corev1.VolumeMount{ + {Name: "kvm", MountPath: "/dev/kvm"}, + {Name: "vhost-net", MountPath: "/dev/vhost-net"}, + {Name: "tun", MountPath: "/dev/net/tun"}, + } + restartAlways := corev1.ContainerRestartPolicyAlways + root := int64(0) + + var containers []corev1.Container + if storage.imageClaim != "" { + containers = append(containers, corev1.Container{ + Name: "copy-images", Image: img.runtime, ImagePullPolicy: img.runtimePull, + Command: []string{"bash", "-ec", "cp -a --reflink=auto " + imageSourcePath + "/. " + fetchPath + "/"}, + Resources: storageReservation(storage.budget), + VolumeMounts: []corev1.VolumeMount{ + {Name: "image-source", MountPath: imageSourcePath, ReadOnly: true}, + {Name: "cvd-images", MountPath: fetchPath}, + }, + }) + } + if storage.fetchImages { + containers = append(containers, corev1.Container{ + Name: "fetch-images", Image: img.runtime, ImagePullPolicy: img.runtimePull, + Command: []string{"cvd", "fetch", "--default_build=" + storage.build, "--target_directory=" + fetchPath}, + Resources: storageReservation(storage.budget), + VolumeMounts: []corev1.VolumeMount{{Name: "cvd-images", MountPath: fetchPath}}, + }) + } + return append(containers, + corev1.Container{ + Name: "fix-cuttlefish-permissions", Image: img.runtime, ImagePullPolicy: img.runtimePull, + Command: []string{"bash", "-c", "mkdir -p " + cvdStatePath + " " + androidTmpPath + + " && chown -R httpcvd:httpcvd " + cvdStatePath + " " + androidTmpPath + " " + fetchPath}, + // chown needs UID 0 regardless of the runtime image's default user. + SecurityContext: &corev1.SecurityContext{RunAsUser: &root}, + VolumeMounts: stateMounts, + }, + corev1.Container{ + Name: runtimeContainerName, Image: img.runtime, ImagePullPolicy: img.runtimePull, + RestartPolicy: &restartAlways, + // The marker lets the exporter and probe detect a sidecar restart that lost runtime state. + Command: []string{"bash", "-ec", "cat /proc/sys/kernel/random/uuid > " + cvdStatePath + "/runtime-id\n" + + "chmod 644 " + cvdStatePath + "/runtime-id\nexec /root/run_services.sh"}, + Resources: runtime, + SecurityContext: &corev1.SecurityContext{Privileged: boolPtr(true), RunAsUser: &root}, + VolumeMounts: append(stateMounts, deviceMounts...), + }, + corev1.Container{ + // Runs in the exporter image so the check shares the network namespace and Python runtime with jmp. + Name: gateContainerName, Image: img.exporter, ImagePullPolicy: img.exporterPull, + Command: []string{"python3", "-m", "jumpstarter_driver_cuttlefish.health", "--wait", hostOrchestratorURL}, + SecurityContext: exporterSecurityContext(), + }, + ) +} + +func volumes(storage storageConfig) []corev1.Volume { + emptyDir := func(name string, size *resource.Quantity) corev1.Volume { + return corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: size}, + }} + } + result := []corev1.Volume{ + emptyDir("cvd-images", &storage.imageSize), + emptyDir("cvd-state", &storage.stateSize), + emptyDir("android-tmp", &storage.tmpSize), + deviceVolume("kvm", "/dev/kvm"), + deviceVolume("vhost-net", "/dev/vhost-net"), + deviceVolume("tun", "/dev/net/tun"), + } + if storage.imageClaim != "" { + // The claim is only ever read; each Pod copies it into its private image volume. + result = append(result, corev1.Volume{Name: "image-source", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: storage.imageClaim, ReadOnly: true}, + }}) + } + return result +} + +func (p *Provisioner) EnrichExporterExport( + drivers []virtualtargetv1alpha1.DriverConfig, + mergedParameters map[string]interface{}, +) ([]virtualtargetv1alpha1.DriverConfig, error) { + enriched, _, err := enrichDrivers(drivers, mergedParameters) + return enriched, err +} + +// enrichDrivers pins every driver to the in-Pod runtime and returns the +// effective guest size the runtime container must budget for. +func enrichDrivers(drivers []virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) ([]virtualtargetv1alpha1.DriverConfig, guestSpec, error) { + var guest guestSpec + found := 0 + result := make([]virtualtargetv1alpha1.DriverConfig, 0, len(drivers)) + for _, driver := range drivers { + var err error + switch driver.Type { + case cuttlefishDriverType: + found++ + driver, guest, err = enrichCuttlefishDriver(driver, parameters) + case netsimDriverType: + driver, err = pinDriverConfig(driver, map[string]interface{}{"host": "127.0.0.1", "port": netsimPort}, "netsim") + case btPeerDriverType: + driver, err = pinDriverConfig(driver, map[string]interface{}{"transport": fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort)}, "bt_peer") + } + if err != nil { + return nil, guest, err + } + result = append(result, driver) + } + if found != 1 { + return nil, guest, fmt.Errorf("cuttlefish requires exactly one Cuttlefish driver per Pod, got %d", found) + } + return result, guest, nil +} + +func enrichCuttlefishDriver(driver virtualtargetv1alpha1.DriverConfig, parameters map[string]interface{}) (virtualtargetv1alpha1.DriverConfig, guestSpec, error) { + var guest guestSpec + var err error + if guest.cpus, err = positiveInt(parameters, "vm_cpus", defaultVMCPUs); err != nil { + return driver, guest, err + } + if guest.memoryMB, err = positiveInt(parameters, "vm_memory_mb", defaultVMMemoryMB); err != nil { + return driver, guest, err + } + config, err := decodeConfig(driver, "Cuttlefish") + if err != nil { + return driver, guest, err + } + + config["managed"] = true + config["health_state_path"] = healthStatePath + config["runtime_id_path"] = runtimeIDPath + config["health_ports"] = healthPorts + // Any other endpoint would bypass the managed runtime in this Pod. + for key, value := range map[string]interface{}{"scheme": "http", "host": "127.0.0.1", "port": hostOrchestratorPort, "instance_num": 1} { + if err := pin(config, key, value); err != nil { + return driver, guest, err + } + } + setDefault(config, "group", "cvd") + setDefault(config, "name", "1") + setDefault(config, "boot_timeout", 300) + + envConfig, err := configObject(config, "env_config") + if err != nil { + return driver, guest, err + } + common, err := configObject(envConfig, "common") + if err != nil { + return driver, guest, err + } + setDefault(common, "host_package", fetchPath) + envConfig["common"] = common + // Standalone RootCanal does not propagate the userspace VSOCK flag. + if err := pin(envConfig, "netsim_bt", true); err != nil { + return driver, guest, fmt.Errorf("%w; standalone RootCanal is not supported", err) + } + + instances, ok := envConfig["instances"].([]interface{}) + if raw, exists := envConfig["instances"]; exists && (!ok || len(instances) != 1) { + return driver, guest, fmt.Errorf("env_config.instances must contain exactly one instance, got %v", raw) + } + if len(instances) == 0 { + instances = []interface{}{map[string]interface{}{}} + } + instance, ok := instances[0].(map[string]interface{}) + if !ok || instance == nil { + return driver, guest, fmt.Errorf("env_config.instances[0] must be an object") + } + disk, err := configObject(instance, "disk") + if err != nil { + return driver, guest, err + } + setDefault(disk, "default_build", fetchPath) + instance["disk"] = disk + graphics, err := configObject(instance, "graphics") + if err != nil { + return driver, guest, err + } + gpuMode := defaultGPUMode + if configured, ok := parameters["gpu_mode"].(string); ok && configured != "" { + gpuMode = configured + } + setDefault(graphics, "gpu_mode", gpuMode) + instance["graphics"] = graphics + + vm, err := configObject(instance, "vm") + if err != nil { + return driver, guest, err + } + for _, other := range []string{"qemu", "gem5"} { + if _, exists := vm[other]; exists { + return driver, guest, fmt.Errorf("managed Cuttlefish requires crosvm with private userspace VSOCK, got vm.%s", other) + } + } + crosvm, err := configObject(vm, "crosvm") + if err != nil { + return driver, guest, err + } + // The upstream schema wants the string "true" here; two Pods on one node share guest CIDs otherwise. + if err := pin(crosvm, "vhost_user_vsock", "true"); err != nil { + return driver, guest, err + } + vm["crosvm"] = crosvm + // Template guest values take precedence over the class parameters. + setDefault(vm, "cpus", guest.cpus) + setDefault(vm, "memory_mb", guest.memoryMB) + if guest.cpus, err = positiveInt(vm, "cpus", guest.cpus); err != nil { + return driver, guest, err + } + if guest.memoryMB, err = positiveInt(vm, "memory_mb", guest.memoryMB); err != nil { + return driver, guest, err + } + instance["vm"] = vm + instances[0] = instance + envConfig["instances"] = instances + config["env_config"] = envConfig + + driver, err = encodeConfig(driver, config) + return driver, guest, err +} + +// pin sets config[key] to value and rejects a template value that differs. +// Values are compared through their JSON encoding so 2081 matches 2081.0. +func pin(config map[string]interface{}, key string, value interface{}) error { + if current, exists := config[key]; exists && !jsonEqual(current, value) { + return fmt.Errorf("managed Cuttlefish requires %s=%v, got %v", key, value, current) + } + config[key] = value + return nil +} + +func jsonEqual(a, b interface{}) bool { + rawA, errA := json.Marshal(a) + rawB, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(rawA, rawB) +} + +func configObject(parent map[string]interface{}, key string) (map[string]interface{}, error) { + raw, exists := parent[key] + if !exists { + return map[string]interface{}{}, nil + } + value, ok := raw.(map[string]interface{}) + if !ok || value == nil { + return nil, fmt.Errorf("%s must be an object", key) + } + return value, nil +} + +// pinDriverConfig pins a sidecar driver to the simulator endpoints this Pod +// runs; a template value that differs points the driver outside the Pod and is +// rejected, while a matching one is preserved. Keys are applied in sorted order +// so a conflicting template yields a stable error. +func pinDriverConfig(driver virtualtargetv1alpha1.DriverConfig, managed map[string]interface{}, name string) (virtualtargetv1alpha1.DriverConfig, error) { + config, err := decodeConfig(driver, name) + if err != nil { + return driver, err + } + for _, key := range slices.Sorted(maps.Keys(managed)) { + if err := pin(config, key, managed[key]); err != nil { + return driver, fmt.Errorf("%s driver: %w", name, err) + } + } + return encodeConfig(driver, config) +} + +func decodeConfig(driver virtualtargetv1alpha1.DriverConfig, name string) (map[string]interface{}, error) { + config := map[string]interface{}{} + if driver.Config != nil && driver.Config.Raw != nil { + if err := json.Unmarshal(driver.Config.Raw, &config); err != nil { + return nil, fmt.Errorf("unmarshal %s driver config: %w", name, err) + } + } + if config == nil { + return nil, fmt.Errorf("%s config must be an object", name) + } + return config, nil +} + +func encodeConfig(driver virtualtargetv1alpha1.DriverConfig, config map[string]interface{}) (virtualtargetv1alpha1.DriverConfig, error) { + raw, err := json.Marshal(config) + if err != nil { + return driver, fmt.Errorf("marshal driver config: %w", err) + } + driver.Config = &apiextensionsv1.JSON{Raw: raw} + return driver, nil +} + +func positiveInt(values map[string]interface{}, key string, fallback int) (int, error) { + raw, exists := values[key] + if !exists { + return fallback, nil + } + var value float64 + switch v := raw.(type) { + case int: + value = float64(v) + case int32: + value = float64(v) + case int64: + value = float64(v) + case float64: + value = v + default: + return 0, fmt.Errorf("%s must be a positive integer", key) + } + if math.IsNaN(value) || math.IsInf(value, 0) || value < 1 || value > math.MaxInt32 || math.Trunc(value) != value { + return 0, fmt.Errorf("%s must be a positive integer", key) + } + return int(value), nil +} + +// runtimeResources starts from the class scheduling resources and guarantees +// the runtime container requests the guest memory plus runtime overhead. +func runtimeResources(vtc *virtualtargetv1alpha1.VirtualTargetClass, guest guestSpec, parameters map[string]interface{}) (corev1.ResourceRequirements, error) { + resources := corev1.ResourceRequirements{} + if vtc.Spec.Scheduling != nil && vtc.Spec.Scheduling.Resources != nil { + resources = *vtc.Spec.Scheduling.Resources.DeepCopy() + } + overhead, err := positiveInt(parameters, "runtime_memory_overhead_mb", defaultOverheadMB) + if err != nil { + return resources, err + } + budget := *resource.NewQuantity(int64(guest.memoryMB+overhead)*1024*1024, resource.BinarySI) + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { + if value, exists := values[corev1.ResourceMemory]; exists && value.Cmp(budget) < 0 { + return resources, fmt.Errorf("runtime memory must be at least %s for guest plus overhead", budget.String()) + } + } + if _, exists := resources.Requests[corev1.ResourceMemory]; !exists { + resources.Requests[corev1.ResourceMemory] = budget + } + if limit, exists := resources.Limits[corev1.ResourceMemory]; exists && resources.Requests.Memory().Cmp(limit) > 0 { + return resources, fmt.Errorf("runtime memory request exceeds limit") + } + if _, exists := resources.Requests[corev1.ResourceCPU]; !exists { + if limit, exists := resources.Limits[corev1.ResourceCPU]; exists { + resources.Requests[corev1.ResourceCPU] = limit.DeepCopy() + } else { + resources.Requests[corev1.ResourceCPU] = *resource.NewQuantity(int64(guest.cpus), resource.DecimalSI) + } + } + if resources.Requests.Cpu().Sign() <= 0 { + return resources, fmt.Errorf("runtime CPU request must be positive") + } + if limit, exists := resources.Limits[corev1.ResourceCPU]; exists && resources.Requests.Cpu().Cmp(limit) > 0 { + return resources, fmt.Errorf("runtime CPU request exceeds limit") + } + return resources, nil +} + +// storageReservation is the ephemeral storage for containers that only touch the volumes. +func storageReservation(budget resource.Quantity) corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: budget.DeepCopy()}, + Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: budget.DeepCopy()}, + } +} + +// reserveStorage adds the ephemeral storage budget to class-provided resources. +func reserveStorage(resources *corev1.ResourceRequirements, budget resource.Quantity) error { + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + for _, values := range []corev1.ResourceList{resources.Requests, resources.Limits} { + if value, exists := values[corev1.ResourceEphemeralStorage]; exists && value.Cmp(budget) < 0 { + return fmt.Errorf("ephemeral-storage must be at least %s for Cuttlefish volume budgets and overhead", budget.String()) + } + } + if _, exists := resources.Requests[corev1.ResourceEphemeralStorage]; !exists { + resources.Requests[corev1.ResourceEphemeralStorage] = budget.DeepCopy() + } + if _, exists := resources.Limits[corev1.ResourceEphemeralStorage]; !exists { + resources.Limits[corev1.ResourceEphemeralStorage] = resources.Requests[corev1.ResourceEphemeralStorage].DeepCopy() + } + request := resources.Requests[corev1.ResourceEphemeralStorage] + if request.Cmp(resources.Limits[corev1.ResourceEphemeralStorage]) > 0 { + return fmt.Errorf("ephemeral-storage request exceeds limit") + } + return nil +} + +func (p *Provisioner) RenderNetworkPolicy(es *virtualtargetv1alpha1.ExporterSet) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "cuttlefish-" + string(es.UID), Namespace: es.Namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{isolationLabel: string(es.UID)}}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + }, + } +} + +func setDefault(config map[string]interface{}, key string, value interface{}) { + if _, exists := config[key]; !exists { + config[key] = value + } +} + +func deviceVolume(name, path string) corev1.Volume { + typeCharDevice := corev1.HostPathCharDev + return corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: path, + Type: &typeCharDevice, + }}, + } +} + +func boolPtr(value bool) *bool { + return &value +} + +func (p *Provisioner) Cleanup( + ctx context.Context, + exporterSet *virtualtargetv1alpha1.ExporterSet, + exporter *jumpstarterdevv1alpha1.Exporter, +) error { + // Cuttlefish state and fetched images are Pod-scoped. Kubernetes removes + // the Pod-owned emptyDir volumes, while a claimed image tree is external + // and must outlive the exporter, so there is nothing for the provisioner + // to clean up here. + return nil +} diff --git a/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go new file mode 100644 index 000000000..a4306f780 --- /dev/null +++ b/controller/internal/exporterset/provisioners/cuttlefish/cuttlefish_test.go @@ -0,0 +1,465 @@ +package cuttlefish + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestProvisionerName(t *testing.T) { + if got := New("dev").Name(); got != ProvisionerName { + t.Fatalf("Name() = %q, want %q", got, ProvisionerName) + } +} + +func TestRenderPod(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + names := make([]string, 0, len(pod.Spec.InitContainers)) + for _, container := range pod.Spec.InitContainers { + names = append(names, container.Name) + } + want := []string{"fetch-images", "fix-cuttlefish-permissions", runtimeContainerName, gateContainerName} + if fmt.Sprint(names) != fmt.Sprint(want) { + t.Fatalf("init containers = %v, want %v", names, want) + } + runtime := initContainer(t, pod, runtimeContainerName) + if runtime.RestartPolicy == nil || *runtime.RestartPolicy != corev1.ContainerRestartPolicyAlways { + t.Fatal("runtime must be a native sidecar") + } + if runtime.SecurityContext == nil || runtime.SecurityContext.Privileged == nil || !*runtime.SecurityContext.Privileged { + t.Fatal("Cuttlefish runtime must be privileged") + } + permissions := initContainer(t, pod, "fix-cuttlefish-permissions") + if permissions.SecurityContext == nil || permissions.SecurityContext.RunAsUser == nil || *permissions.SecurityContext.RunAsUser != 0 { + t.Fatal("chown init container must run as UID 0") + } + if len(pod.Spec.Containers) != 1 || pod.Spec.Containers[0].Name != "exporter" { + t.Fatalf("containers = %#v", pod.Spec.Containers) + } + exporter := pod.Spec.Containers[0] + if !hasEnv(exporter.Env, "HOME", "/tmp") { + t.Errorf("exporter HOME = %#v, want /tmp", exporter.Env) + } + if exporter.Command[6] != hostOrchestratorURL { + t.Errorf("exporter endpoint = %q", exporter.Command[6]) + } + if pod.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Error("Pod must not restart the exporter in place") + } + if !hasVolume(pod.Spec.Volumes, "kvm", "/dev/kvm") || !hasVolume(pod.Spec.Volumes, "tun", "/dev/net/tun") { + t.Fatalf("device volumes missing: %#v", pod.Spec.Volumes) + } +} + +func TestRenderPod_rejectsFetchingIntoClaim(t *testing.T) { + _, err := New("dev").RenderPod(context.Background(), testExporterSet(), &virtualtargetv1alpha1.VirtualTargetClass{}, map[string]interface{}{ + "fetch_images": true, + "image_volume_claim": "cuttlefish-images", + "runtime_privileged": true, + "service_account_name": "cuttlefish-runtime", + }, nil, nil) + if err == nil { + t.Fatal("RenderPod() succeeded; want an error for fetch_images with image_volume_claim") + } +} + +func TestRenderPod_privateImageCopy(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"image_volume_claim": "images"}) + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && !volume.PersistentVolumeClaim.ReadOnly { + t.Fatal("source claim is writable") + } + if volume.Name == "cvd-images" && volume.EmptyDir == nil { + t.Fatal("missing private image volume") + } + } + copy := pod.Spec.InitContainers[0] + if copy.Name != "copy-images" || !copy.VolumeMounts[0].ReadOnly || copy.VolumeMounts[1].ReadOnly { + t.Fatal("invalid copy mounts") + } + for _, container := range pod.Spec.InitContainers[1:] { + for _, mount := range container.VolumeMounts { + if mount.Name == "image-source" { + t.Fatalf("%s can access shared source", container.Name) + } + } + } +} + +func TestRenderPod_healthGate(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + gate := pod.Spec.InitContainers[len(pod.Spec.InitContainers)-1] + if gate.Name != gateContainerName || gate.Command[3] != "--wait" || gate.Command[4] != hostOrchestratorURL { + t.Fatalf("missing API startup gate: %#v", gate.Command) + } + if gate.Image != pod.Spec.Containers[0].Image { + t.Fatal("gate must run in the exporter image") + } + probe := pod.Spec.Containers[0].LivenessProbe + if probe == nil || probe.Exec.Command[2] != "jumpstarter_driver_cuttlefish.health" || probe.Exec.Command[3] != healthStatePath { + t.Fatal("missing runtime failure detection") + } + if !hasMount(pod.Spec.Containers[0].VolumeMounts, "cvd-state", runtimeIDMount) { + t.Fatal("exporter cannot read the runtime marker") + } +} + +func TestRenderPod_storageBudgets(t *testing.T) { + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true, "storage": map[string]interface{}{"imageSize": "8Gi", "stateSize": "4Gi", "tmpSize": "2Gi"}}) + total := resource.MustParse("1Gi") + for _, volume := range pod.Spec.Volumes { + if volume.EmptyDir != nil { + if volume.EmptyDir.SizeLimit == nil { + t.Fatalf("%s has no budget", volume.Name) + } + total.Add(*volume.EmptyDir.SizeLimit) + } + } + if total.Cmp(resource.MustParse("15Gi")) != 0 { + t.Fatalf("volume total = %s", total.String()) + } + for _, name := range []string{"fetch-images", runtimeContainerName} { + container := initContainer(t, pod, name) + request := container.Resources.Requests[corev1.ResourceEphemeralStorage] + limit := container.Resources.Limits[corev1.ResourceEphemeralStorage] + if request.Cmp(total) != 0 || limit.Cmp(total) != 0 { + t.Fatalf("%s storage does not cover volumes: %v", name, container.Resources) + } + } +} + +func TestStorageValidation(t *testing.T) { + for _, value := range []interface{}{"", "0", "-1Gi", "invalid", 42} { + _, err := resolveStorageConfig(map[string]interface{}{"fetch_images": true, "storage": map[string]interface{}{"imageSize": value}}) + if err == nil { + t.Fatalf("accepted invalid size %v", value) + } + } + if _, err := resolveStorageConfig(map[string]interface{}{"fetch_images": true, "storage": "invalid"}); err == nil { + t.Fatal("accepted non-object storage") + } + budget := resource.MustParse("10Gi") + for _, resources := range []corev1.ResourceRequirements{ + {Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, + {Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("1Gi")}}, + {Requests: corev1.ResourceList{corev1.ResourceEphemeralStorage: resource.MustParse("20Gi")}, Limits: corev1.ResourceList{corev1.ResourceEphemeralStorage: budget}}, + } { + if reserveStorage(&resources, budget) == nil { + t.Fatal("accepted insufficient or inconsistent storage") + } + } +} + +func TestEnrichExporterExport(t *testing.T) { + drivers := []virtualtargetv1alpha1.DriverConfig{ + {Name: "cuttlefish", Type: cuttlefishDriverType}, + {Name: "netsim", Type: netsimDriverType}, + {Name: "bt_peer", Type: btPeerDriverType}, + } + result, err := New("dev").EnrichExporterExport(drivers, map[string]interface{}{ + "default_build": "aosp/test", + "gpu_mode": "none", + }) + if err != nil { + t.Fatal(err) + } + + cuttlefish := configFor(t, result[0]) + if cuttlefish["host"] != "127.0.0.1" || cuttlefish["port"] != float64(hostOrchestratorPort) || cuttlefish["scheme"] != "http" { + t.Errorf("Cuttlefish endpoint = %#v", cuttlefish) + } + if fmt.Sprint(cuttlefish["health_ports"]) != fmt.Sprint([]interface{}{float64(netsimPort), float64(hciPort)}) { + t.Errorf("health_ports = %v", cuttlefish["health_ports"]) + } + envConfig := cuttlefish["env_config"].(map[string]interface{}) + instance := envConfig["instances"].([]interface{})[0].(map[string]interface{}) + graphics := instance["graphics"].(map[string]interface{}) + if graphics["gpu_mode"] != "none" { + t.Errorf("gpu_mode = %v", graphics["gpu_mode"]) + } + vm := instance["vm"].(map[string]interface{}) + if vm["cpus"] != float64(defaultVMCPUs) || vm["memory_mb"] != float64(defaultVMMemoryMB) { + t.Errorf("vm config = %#v", vm) + } + + netsim := configFor(t, result[1]) + if netsim["host"] != "127.0.0.1" || netsim["port"] != float64(netsimPort) { + t.Errorf("netsim config = %#v", netsim) + } + btPeer := configFor(t, result[2]) + if btPeer["transport"] != fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort) { + t.Errorf("bt_peer config = %#v", btPeer) + } +} + +func TestEnrichExporterExportDefaultsPodSafeGraphicsAndVM(t *testing.T) { + result, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{ + {Name: "cuttlefish", Type: cuttlefishDriverType}, + }, nil) + if err != nil { + t.Fatal(err) + } + config := configFor(t, result[0]) + envConfig := config["env_config"].(map[string]interface{}) + instance := envConfig["instances"].([]interface{})[0].(map[string]interface{}) + graphics := instance["graphics"].(map[string]interface{}) + if graphics["gpu_mode"] != defaultGPUMode { + t.Errorf("gpu_mode = %v, want %q", graphics["gpu_mode"], defaultGPUMode) + } +} + +func TestEnrichExporterExportRejectsExternalEndpoints(t *testing.T) { + for _, config := range []map[string]interface{}{{"host": "custom-host"}, {"port": 9999}, {"instance_num": 2}, {"scheme": "https"}} { + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(config)} + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{driver}, nil); err == nil { + t.Errorf("accepted external endpoint %v", config) + } + } + // Restating the pinned values is fine, including as JSON numbers. + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(map[string]interface{}{ + "host": "127.0.0.1", "port": 2081.0, "instance_num": 1, "scheme": "http", + })} + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{driver}, nil); err != nil { + t.Fatal(err) + } +} + +func TestEnrichExporterExportPinsSimulatorEndpoints(t *testing.T) { + cuttlefish := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType} + for _, driver := range []virtualtargetv1alpha1.DriverConfig{ + {Name: "netsim", Type: netsimDriverType, Config: mustJSON(map[string]interface{}{"host": "netsim.example.com"})}, + {Name: "netsim", Type: netsimDriverType, Config: mustJSON(map[string]interface{}{"port": 9999})}, + {Name: "bt_peer", Type: btPeerDriverType, Config: mustJSON(map[string]interface{}{"transport": "tcp-client:10.0.0.1:7300"})}, + } { + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{cuttlefish, driver}, nil); err == nil { + t.Errorf("accepted external %s endpoint", driver.Name) + } + } + // Restating the in-Pod endpoints, and unrelated keys, stay accepted. + drivers := []virtualtargetv1alpha1.DriverConfig{ + cuttlefish, + {Name: "netsim", Type: netsimDriverType, Config: mustJSON(map[string]interface{}{"host": "127.0.0.1", "port": 7681.0})}, + {Name: "bt_peer", Type: btPeerDriverType, Config: mustJSON(map[string]interface{}{"address": "00:11:22:33:44:55"})}, + } + result, err := New("dev").EnrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + if btPeer := configFor(t, result[2]); btPeer["address"] != "00:11:22:33:44:55" || + btPeer["transport"] != fmt.Sprintf("tcp-client:127.0.0.1:%d", hciPort) { + t.Errorf("bt_peer config = %#v", btPeer) + } +} + +func TestManagedContract(t *testing.T) { + for _, config := range []map[string]interface{}{ + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{}, map[string]interface{}{}}}}, + {"env_config": map[string]interface{}{"instances": "invalid"}}, + {"env_config": map[string]interface{}{"instances": []interface{}{}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{nil}}}, + {"env_config": "invalid"}, + {"env_config": map[string]interface{}{"netsim_bt": false}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"memory_mb": -1}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"crosvm": map[string]interface{}{"vhost_user_vsock": "false"}}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"qemu": map[string]interface{}{}}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"gem5": map[string]interface{}{}}}}}}, + {"env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"crosvm": map[string]interface{}{"vhost_user_vsock": true}}}}}}, + } { + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(config)} + if _, err := New("dev").EnrichExporterExport([]virtualtargetv1alpha1.DriverConfig{driver}, nil); err == nil { + t.Errorf("accepted %v", config) + } + } + drivers := testExporterSet().Spec.Template.Spec.Drivers + if _, err := New("dev").EnrichExporterExport(nil, nil); err == nil { + t.Fatal("accepted missing Cuttlefish driver") + } + if _, err := New("dev").EnrichExporterExport(append(drivers, drivers[0]), nil); err == nil { + t.Fatal("accepted multiple Cuttlefish drivers") + } + for _, params := range []map[string]interface{}{{"vm_memory_mb": 12.5}, {"vm_cpus": "4"}, {"vm_cpus": 0}} { + if _, err := New("dev").EnrichExporterExport(drivers, params); err == nil { + t.Fatalf("accepted guest parameters %v", params) + } + } + enriched, err := New("dev").EnrichExporterExport(drivers, nil) + if err != nil { + t.Fatal(err) + } + config := configFor(t, enriched[0]) + envConfig := config["env_config"].(map[string]interface{}) + vm := envConfig["instances"].([]interface{})[0].(map[string]interface{})["vm"].(map[string]interface{}) + if config["managed"] != true || envConfig["netsim_bt"] != true || vm["crosvm"].(map[string]interface{})["vhost_user_vsock"] != "true" { + t.Fatalf("missing managed isolation: %v", config) + } +} + +func TestGuestSpecPrefersTemplateValues(t *testing.T) { + driver := virtualtargetv1alpha1.DriverConfig{Name: "cuttlefish", Type: cuttlefishDriverType, Config: mustJSON(map[string]interface{}{ + "env_config": map[string]interface{}{"instances": []interface{}{map[string]interface{}{"vm": map[string]interface{}{"cpus": 2}}}}, + })} + _, guest, err := enrichDrivers([]virtualtargetv1alpha1.DriverConfig{driver}, map[string]interface{}{"vm_cpus": 8, "vm_memory_mb": 4096}) + if err != nil { + t.Fatal(err) + } + if guest != (guestSpec{cpus: 2, memoryMB: 4096}) { + t.Fatalf("guest = %+v", guest) + } +} + +func TestRuntimeMemory(t *testing.T) { + for _, tc := range []struct { + name, memory, request, limit, want string + invalid bool + }{ + {name: "default", want: "10Gi"}, + {name: "small limit", limit: "1Gi", invalid: true}, + {name: "small request", request: "8Gi", invalid: true}, + {name: "override", memory: "16384", want: "18Gi"}, + {name: "override limit", memory: "16384", limit: "10Gi", invalid: true}, + {name: "request exceeds limit", request: "12Gi", limit: "10Gi", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + es := testExporterSet() + if tc.memory != "" { + es.Spec.Template.Spec.Drivers[0].Config = &apiextensionsv1.JSON{Raw: []byte(`{"env_config":{"instances":[{"vm":{"memory_mb":` + tc.memory + `}}]}}`)} + } + resources := &corev1.ResourceRequirements{Requests: corev1.ResourceList{}, Limits: corev1.ResourceList{}} + if tc.request != "" { + resources.Requests[corev1.ResourceMemory] = resource.MustParse(tc.request) + } + if tc.limit != "" { + resources.Limits[corev1.ResourceMemory] = resource.MustParse(tc.limit) + } + vtc := &virtualtargetv1alpha1.VirtualTargetClass{Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Scheduling: &virtualtargetv1alpha1.SchedulingSpec{Resources: resources}}} + pod, err := New("dev").RenderPod(context.Background(), es, vtc, map[string]interface{}{"runtime_privileged": true, "service_account_name": "cuttlefish-runtime", "fetch_images": true}, nil, nil) + if tc.invalid { + if err == nil { + t.Fatal("accepted invalid resources") + } + return + } + if err != nil { + t.Fatal(err) + } + runtime := initContainer(t, pod, runtimeContainerName) + if runtime.Resources.Requests.Memory().Cmp(resource.MustParse(tc.want)) != 0 { + t.Fatalf("memory = %s, want %s", runtime.Resources.Requests.Memory(), tc.want) + } + }) + } +} + +func TestPodIsolation(t *testing.T) { + es := testExporterSet() + pod := renderTestPod(t, map[string]interface{}{"fetch_images": true}) + policy := New("dev").RenderNetworkPolicy(es) + if policy.Spec.PodSelector.MatchLabels[isolationLabel] != pod.Labels[isolationLabel] || len(policy.Spec.Ingress) != 0 || len(policy.Spec.PolicyTypes) != 1 || policy.Spec.PolicyTypes[0] != "Ingress" { + t.Fatalf("incorrect isolation policy: %#v", policy.Spec) + } + if pod.Spec.ServiceAccountName != "cuttlefish-runtime" || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("workload service account not isolated") + } + for _, volume := range pod.Spec.Volumes { + if volume.HostPath != nil && volume.HostPath.Path == "/dev/vhost-vsock" { + t.Fatal("kernel VSOCK device mounted") + } + } + for _, params := range []map[string]interface{}{ + {"fetch_images": true, "service_account_name": "cuttlefish-runtime", "runtime_privileged": false}, + {"fetch_images": true, "service_account_name": "cuttlefish-runtime"}, + {"fetch_images": true, "runtime_privileged": true}, + {"fetch_images": true, "runtime_privileged": true, "service_account_name": "default"}, + {"fetch_images": true, "runtime_privileged": true, "service_account_name": "Invalid_Name"}, + } { + if _, err := New("dev").RenderPod(context.Background(), es, &virtualtargetv1alpha1.VirtualTargetClass{}, params, nil, nil); err == nil { + t.Fatalf("accepted %v", params) + } + } + es.Spec.RecycleStrategy = virtualtargetv1alpha1.RecycleStrategyInPlaceReuse + if _, err := New("dev").RenderPod(context.Background(), es, &virtualtargetv1alpha1.VirtualTargetClass{}, map[string]interface{}{ + "fetch_images": true, "runtime_privileged": true, "service_account_name": "cuttlefish-runtime", + }, nil, nil); err == nil { + t.Fatal("accepted InPlaceReuse") + } +} + +func renderTestPod(t *testing.T, params map[string]interface{}) *corev1.Pod { + t.Helper() + params["runtime_privileged"] = true + params["service_account_name"] = "cuttlefish-runtime" + pod, err := New("dev").RenderPod(context.Background(), testExporterSet(), &virtualtargetv1alpha1.VirtualTargetClass{}, params, nil, nil) + if err != nil { + t.Fatal(err) + } + return pod +} + +func initContainer(t *testing.T, pod *corev1.Pod, name string) corev1.Container { + t.Helper() + for _, container := range pod.Spec.InitContainers { + if container.Name == name { + return container + } + } + t.Fatalf("init container %q missing", name) + return corev1.Container{} +} + +func configFor(t *testing.T, driver virtualtargetv1alpha1.DriverConfig) map[string]interface{} { + t.Helper() + var config map[string]interface{} + if err := json.Unmarshal(driver.Config.Raw, &config); err != nil { + t.Fatal(err) + } + return config +} + +func hasVolume(volumes []corev1.Volume, name, path string) bool { + for _, volume := range volumes { + if volume.Name == name && volume.HostPath != nil && volume.HostPath.Path == path { + return true + } + } + return false +} + +func hasMount(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} + +func hasEnv(env []corev1.EnvVar, name, value string) bool { + for _, variable := range env { + if variable.Name == name && variable.Value == value { + return true + } + } + return false +} + +func mustJSON(value interface{}) *apiextensionsv1.JSON { + raw, err := json.Marshal(value) + if err != nil { + panic(err) + } + return &apiextensionsv1.JSON{Raw: raw} +} + +func testExporterSet() *virtualtargetv1alpha1.ExporterSet { + return &virtualtargetv1alpha1.ExporterSet{ + ObjectMeta: metav1.ObjectMeta{Name: "cuttlefish", Namespace: "default", UID: "test-uid"}, + Spec: virtualtargetv1alpha1.ExporterSetSpec{Template: virtualtargetv1alpha1.ExporterSetTemplate{Spec: virtualtargetv1alpha1.ExporterTemplateSpec{Drivers: []virtualtargetv1alpha1.DriverConfig{{Name: "cuttlefish", Type: cuttlefishDriverType}}}}}, + } +} diff --git a/controller/internal/exporterset/reconciler.go b/controller/internal/exporterset/reconciler.go index 8491e875b..843800655 100644 --- a/controller/internal/exporterset/reconciler.go +++ b/controller/internal/exporterset/reconciler.go @@ -40,6 +40,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -88,6 +89,7 @@ type ExporterSetReconciler struct { LastScaleDownAction map[types.NamespacedName]time.Time } +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=virtualtarget.jumpstarter.dev,resources=exportersets/finalizers,verbs=update @@ -173,6 +175,10 @@ func (r *ExporterSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if err := r.syncNetworkPolicy(ctx, &exporterSet); err != nil { + return ctrl.Result{}, err + } + logger.Info("reconciling ExporterSet", "name", exporterSet.Name, "provisioner", vtc.Spec.Provisioner, @@ -288,6 +294,34 @@ func (r *ExporterSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result, nil } +// NetworkPolicyProvisioner isolates backend listeners before any workload Pod is created. +type NetworkPolicyProvisioner interface { + RenderNetworkPolicy(*virtualtargetv1alpha1.ExporterSet) *networkingv1.NetworkPolicy +} + +func (r *ExporterSetReconciler) syncNetworkPolicy(ctx context.Context, es *virtualtargetv1alpha1.ExporterSet) error { + provisioner, ok := r.Provisioner.(NetworkPolicyProvisioner) + if !ok { + return nil + } + desired := provisioner.RenderNetworkPolicy(es) + policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, policy, func() error { + if !policy.CreationTimestamp.IsZero() && !metav1.IsControlledBy(policy, es) { + return fmt.Errorf("network policy %s is not owned by ExporterSet", policy.Name) + } + if err := ctrl.SetControllerReference(es, policy, r.Scheme); err != nil { + return err + } + policy.Spec = *desired.Spec.DeepCopy() + return nil + }) + if err != nil { + return fmt.Errorf("ensure runtime network isolation: %w", err) + } + return nil +} + type poolState struct { replicas int32 ready int32 @@ -1267,7 +1301,7 @@ func filterOwnedExporters( // SetupWithManager sets up the controller with the Manager. func (r *ExporterSetReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + builder := ctrl.NewControllerManagedBy(mgr). For(&virtualtargetv1alpha1.ExporterSet{}). Owns(&jumpstarterdevv1alpha1.Exporter{}). Watches( @@ -1282,8 +1316,11 @@ func (r *ExporterSetReconciler) SetupWithManager(mgr ctrl.Manager) error { &jumpstarterdevv1alpha1.Lease{}, handler.EnqueueRequestsFromMapFunc(r.findExporterSetsForLease), ). - Named("exporterset"). - Complete(r) + Named("exporterset") + if _, ok := r.Provisioner.(NetworkPolicyProvisioner); ok { + builder = builder.Owns(&networkingv1.NetworkPolicy{}) + } + return builder.Complete(r) } // findExporterSetForPod bridges the ExporterSet→Exporter→Pod grandchild gap via label. diff --git a/python/packages/jumpstarter-driver-cuttlefish/README.md b/python/packages/jumpstarter-driver-cuttlefish/README.md index 5f4050108..6a05f5142 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/README.md +++ b/python/packages/jumpstarter-driver-cuttlefish/README.md @@ -6,8 +6,11 @@ virtual devices through the [Host Orchestrator](https://github.com/google/android-cuttlefish) REST API. It provides full CVD (Cuttlefish Virtual Device) lifecycle management through standard Jumpstarter interfaces: `VirtualPowerInterface` for on/off/cycle, -plus cuttlefish-specific operations -(snapshot, powerwash, restart). +plus cuttlefish-specific operations (powerwash and restart). + +For managed Kubernetes exporters, see the [Cuttlefish ExporterSet deployment guide](https://github.com/jumpstarter-dev/jumpstarter/blob/main/controller/internal/exporterset/provisioners/cuttlefish/README.md) +for service accounts/SCCs, network isolation, resource budgets, pinned images, +private VSOCK, storage access modes, and failure recovery. ## Installation @@ -114,11 +117,10 @@ are preserved. ### Teardown -Delete CVDs and snapshots when done to avoid accumulation: +Delete CVDs when done to avoid accumulation: ```bash j power off --destroy # deletes the CVD -j cuttlefish snapshot delete # remove specific snapshots ``` ## Configuration @@ -137,11 +139,8 @@ export: instances: - disk: default_build: /home/vsoc-01/fetch - vm: - enable_virtiofs: false # required for snapshot support common: host_package: /home/vsoc-01/fetch - gpu_mode: guest_swiftshader # required for snapshot support netsim: type: jumpstarter_driver_netsim.driver.Netsim config: @@ -220,10 +219,6 @@ j cuttlefish powerbtn # List running operations j cuttlefish ops -# Snapshot management -# Requires: x86_64 host, enable_virtiofs: false, gpu_mode: guest_swiftshader -j cuttlefish snapshot create --id my-snapshot -j cuttlefish snapshot delete ``` ### Python API @@ -251,9 +246,6 @@ with serve(driver) as client: cvds = client.list_cvds() print(cvds) - # Snapshots - client.create_snapshot(snapshot_id="baseline") - # Cleanup client.power.off(destroy=True) ``` diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py index 42ec00deb..e92037121 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver.py @@ -1,8 +1,11 @@ import json +import os import subprocess +import threading import time from collections.abc import Generator from dataclasses import dataclass, field +from pathlib import Path import requests from jumpstarter_driver_adb.driver import AdbServer @@ -39,16 +42,44 @@ class Cuttlefish(Driver): boot_timeout: int = 300 env_config: dict = field(default_factory=dict) webrtc_url: str = "" + managed: bool = False + health_state_path: str = "" + runtime_id_path: str = "" + health_ports: list[int] = field(default_factory=list) + _operation_lock: threading.RLock = field(default_factory=threading.RLock, init=False, repr=False) + _health: dict = field(default_factory=dict, init=False, repr=False) _cvd_group: str | None = field(default=None, init=False, repr=False) _cvd_name: str | None = field(default=None, init=False, repr=False) def __post_init__(self): if hasattr(super(), "__post_init__"): super().__post_init__() + if self.managed: + self._validate_managed_config() + self._health = json.loads(Path(self.health_state_path).read_text()) + if self._health["runtime_id"] != Path(self.runtime_id_path).read_text().strip(): + raise CuttlefishError("Cuttlefish runtime restarted before driver initialization") + self._health.update(url=self._base_url, ports=self.health_ports) + self._write_health() + self.children["power"] = CvdPower(parent=self) self.children["storage"] = CvdFlasher(parent=self) self.children["adb"] = AdbServer(host="127.0.0.1", port=self.adb_server_port) + def _validate_managed_config(self): + instances = self.env_config.get("instances", []) + if len(instances) != 1 or not isinstance(instances[0], dict): + raise CuttlefishError("managed Cuttlefish requires exactly one instance") + if not self.health_state_path or not self.runtime_id_path: + raise CuttlefishError("managed Cuttlefish requires health and runtime ID paths") + + def _write_health(self): + if not self.managed: + return + temporary = Path(self.health_state_path + ".tmp") + temporary.write_text(json.dumps(self._health)) + os.replace(temporary, self.health_state_path) + @classmethod def client(cls) -> str: return "jumpstarter_driver_cuttlefish.client.CuttlefishClient" @@ -125,7 +156,51 @@ def _wait_for_operation(self, op_name: str, timeout: float = 300) -> dict: return r.json() raise CuttlefishTimeout(f"operation {op_name} timed out after {timeout}s") - def _do_operation( + def _validate_creation(self, data): + # Accept only the budgeted, provisioner-approved configuration. Alternate + # HO creation forms and additional groups would bypass the Pod contract. + if data != {"env_config": self.env_config}: + raise CuttlefishError("managed create_cvd requires the configured env_config") + existing = self._request("GET", "/cvds") + if not isinstance(existing, dict) or not isinstance(existing.get("cvds"), list): + raise CuttlefishError("invalid CVD inventory; refusing creation") + if existing["cvds"]: + raise CuttlefishError("managed Cuttlefish already has a CVD; destroy it before creating another") + + def _do_operation(self, method: str, path: str, data: dict | None = None, timeout: float = 300): + if not self.managed: + return self._perform_operation(method, path, data, timeout) + with self._operation_lock: + if method == "POST" and path == "/cvds": + self._validate_creation(data) + state = self._health.get("state", "off") + target_state = state + if path == "/reset" or method == "DELETE" or path.endswith("/:stop"): + target_state = "off" + elif path == "/cvds" or path.endswith(("/:start", "/:restart", "/:powerwash")): + target_state = "running" + # Probes allow bounded transitions, then fail closed if the operation + # hangs or leaves the target's state unknown. + self._health.update(state="transition", deadline=time.monotonic() + timeout + 30) + self._write_health() + try: + result = self._perform_operation(method, path, data, timeout) + except Exception: + self._health["state"] = "failed" + self._write_health() + raise + if isinstance(result, dict) and path == "/cvds": + cvds = result.get("cvds", []) + if len(cvds) == 1: + self._cvd_group = cvds[0].get("group") + self._cvd_name = cvds[0].get("name") + self._health.update( + state=target_state, group=self._cvd_group or self.group, name=self._cvd_name or self.name, + ) + self._write_health() + return result + + def _perform_operation( self, method: str, path: str, @@ -295,7 +370,7 @@ def create_cvd(self, config_json: str) -> str: @export def start_cvd(self) -> str: - return self._fmt(self._do_operation("POST", f"{self._cvd_path}/:start")) + return self._fmt(self._do_operation("POST", f"{self._cvd_path}/:start", {})) @export def stop_cvd(self) -> str: @@ -358,9 +433,16 @@ def client(cls) -> str: return "jumpstarter_driver_cuttlefish.client.CvdPowerClient" @export - def on(self) -> None: # noqa: C901 + def on(self) -> None: + with self.parent._operation_lock: + self._on() + + def _on(self) -> None: # noqa: C901 existing = self.parent._get_existing_cvds() + if self.parent.managed and len(existing) > 1: + raise CuttlefishError("managed Cuttlefish inventory has multiple CVDs") + if len(existing) > 1: self.logger.warning( "Found %d stale CVDs in group %s, deleting", len(existing), self.parent._cvd_group or self.parent.group @@ -392,7 +474,7 @@ def on(self) -> None: # noqa: C901 cvd.get("status"), ) if cvd.get("status") != "Running": - self.parent._do_operation("POST", f"{self.parent._cvd_path}/:start") + self.parent.start_cvd() else: self.logger.info("Creating CVD from env_config") try: @@ -426,12 +508,21 @@ def on(self) -> None: # noqa: C901 ) break + if self.parent.managed: + with self.parent._operation_lock: + self.parent._health.update(state="running", group=self.parent._cvd_group or self.parent.group, + name=self.parent._cvd_name or self.parent.name) + self.parent._write_health() self.parent._auto_connect_adb() if self.parent.boot_timeout: self.parent._wait_boot(self.parent.boot_timeout) @export def off(self, destroy: bool = False) -> None: + with self.parent._operation_lock: + self._off(destroy) + + def _off(self, destroy: bool) -> None: p = self.parent cvd_id = f"{p._cvd_group or p.group}/{p._cvd_name or p.name}" if destroy: diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py index 50207d0dd..4a0458241 100644 --- a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/driver_test.py @@ -149,6 +149,7 @@ def test_start_cvd(requests_mock, drv): _mock_op(requests_mock, "post", "/cvds/cvd_1/dev1/:start") result = json.loads(drv.start_cvd()) assert result["done"] is True + assert requests_mock.request_history[0].json() == {} def test_stop_cvd(requests_mock, drv): @@ -404,6 +405,8 @@ def test_cvd_power_on_existing_stopped(requests_mock, drv): requests_mock.post(f"{BASE}/cvds/cvd_1/dev1/:start", json={"name": "op-1", "done": False}) requests_mock.post(f"{BASE}/operations/op-1/:wait", json={"name": "op-1", "done": True}) power.on() + start_request = next(r for r in requests_mock.request_history if r.url.endswith("/:start")) + assert start_request.json() == {} def test_cvd_power_on_create_new(requests_mock, drv): @@ -517,3 +520,102 @@ def test_cvd_power_on_ignores_other_groups(requests_mock, drv): assert drv._cvd_group == "cvd_1" assert drv._cvd_name == "dev1" assert not any(r.method == "DELETE" for r in requests_mock.request_history) + + +@pytest.fixture +def managed_drv(drv, tmp_path): + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("first-runtime") + drv.managed = True + drv.runtime_id_path = str(runtime_id) + drv.health_state_path = str(tmp_path / "health.json") + drv.env_config = {"instances": [{"vm": {"memory_mb": 8192}}]} + drv._health = {"state": "off", "runtime_id": "first-runtime", "runtime_id_path": str(runtime_id)} + drv._write_health() + return drv + + +def test_managed_rejects_alternate_creation(requests_mock, managed_drv): + for config in ({}, {"env_config": {"instances": [{}, {}]}}, {"cvd": {}}, {"env_config": {}}): + with pytest.raises(CuttlefishError, match="configured env_config"): + managed_drv.create_cvd(json.dumps(config)) + assert requests_mock.call_count == 0 + + +def test_managed_creation_checks_all_groups(requests_mock, managed_drv): + requests_mock.get(f"{BASE}/cvds", json={"cvds": [{"group": "other", "name": "other"}]}) + with pytest.raises(CuttlefishError, match="already has a CVD"): + managed_drv.create_cvd(json.dumps({"env_config": managed_drv.env_config})) + assert all(r.method == "GET" for r in requests_mock.request_history) + + +def test_managed_power_creation_checks_all_groups(requests_mock, managed_drv): + requests_mock.get(f"{BASE}/cvds", json={"cvds": [{"group": "other", "name": "other"}]}) + with pytest.raises(CuttlefishError, match="already has a CVD"): + managed_drv.children["power"].on() + assert all(r.method == "GET" for r in requests_mock.request_history) + + +def test_managed_serializes_creation(managed_drv): + from concurrent.futures import ThreadPoolExecutor + + inventory = [] + + def create(*args): + inventory.append({"group": "cvd_1", "name": "dev1"}) + return {"cvds": inventory} + + def attempt(): + try: + managed_drv.create_cvd(json.dumps({"env_config": managed_drv.env_config})) + return True + except CuttlefishError: + return False + + with patch.object(managed_drv, "_request", side_effect=lambda *args: {"cvds": inventory}), \ + patch.object(managed_drv, "_perform_operation", side_effect=create), ThreadPoolExecutor(2) as pool: + assert sorted(pool.map(lambda _: attempt(), range(2))) == [False, True] + assert len(inventory) == 1 + + +@pytest.mark.parametrize("operation,expected", [ + ("start_cvd", "running"), ("stop_cvd", "off"), ("delete_cvd", "off"), + ("restart_cvd", "running"), ("powerwash_cvd", "running"), ("reset_host", "off"), +]) +def test_managed_records_power_intent(managed_drv, operation, expected): + from pathlib import Path + + def perform(*args): + assert json.loads(Path(managed_drv.health_state_path).read_text())["state"] == "transition" + return {} + + with patch.object(managed_drv, "_perform_operation", side_effect=perform): + getattr(managed_drv, operation)() + assert json.loads(Path(managed_drv.health_state_path).read_text())["state"] == expected + + +def test_managed_records_failed_operation(managed_drv): + from pathlib import Path + + with patch.object(managed_drv, "_perform_operation", side_effect=CuttlefishError("runtime died")): + with pytest.raises(CuttlefishError): + managed_drv.start_cvd() + assert json.loads(Path(managed_drv.health_state_path).read_text())["state"] == "failed" + + +def test_managed_initialization_preserves_startup_runtime_id(drv, tmp_path): + from .health import initialize + + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("runtime-1") + state_path = tmp_path / "health.json" + initialize(str(state_path), str(runtime_id), BASE) + config = {"managed": True, "runtime_id_path": str(runtime_id), "health_state_path": str(state_path), + "env_config": {"instances": [{}]}, "health_ports": [7681]} + instance = Cuttlefish(**config) + assert instance._health["runtime_id"] == "runtime-1" + assert instance._health["ports"] == [7681] + instance.close() + runtime_id.write_text("runtime-2") + with pytest.raises(CuttlefishError, match="restarted"): + Cuttlefish(**config) diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py new file mode 100644 index 000000000..d8112e080 --- /dev/null +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health.py @@ -0,0 +1,79 @@ +"""Pod liveness check using driver intent rather than requiring an always-on guest.""" + +import json +import os +import sys +import time +import urllib.request +from pathlib import Path + + +def initialize(state_path: str, runtime_id_path: str, url: str) -> None: + runtime_id = Path(runtime_id_path).read_text().strip() + if not runtime_id: + raise RuntimeError("Cuttlefish runtime ID is empty") + Path(state_path).write_text(json.dumps({ + "runtime_id_path": runtime_id_path, "runtime_id": runtime_id, + "url": url, "ports": [], "state": "off", + })) + + +def check(state_path: str) -> None: + state = json.loads(Path(state_path).read_text()) + runtime_id = Path(state["runtime_id_path"]).read_text().strip() + if not runtime_id or runtime_id != state["runtime_id"]: + raise RuntimeError("Cuttlefish runtime restarted; release the lease to replace this Pod") + url = state["url"] + with urllib.request.urlopen(f"{url}/_debug/statusz", timeout=2): + pass + if state["state"] == "off": + return + if state["state"] == "transition" and time.monotonic() < state["deadline"]: + return + if state["state"] != "running": + raise RuntimeError("Cuttlefish operation failed or timed out") + with urllib.request.urlopen(f"{url}/cvds", timeout=2) as response: + cvds = json.load(response)["cvds"] + if len(cvds) != 1 or any(cvds[0].get(key) != state[key] for key in ("group", "name")): + raise RuntimeError("Cuttlefish inventory no longer matches this exporter") + if cvds[0].get("status") != "Running": + raise RuntimeError("CVD stopped unexpectedly") + if not set(state["ports"]).issubset(listening_ports()): + raise RuntimeError("Cuttlefish simulator listener is missing") + + +def wait_ready(url: str, attempts: int = 60, interval: float = 5) -> None: + """Startup gate: block until Host Orchestrator answers, so the exporter never registers early.""" + for _ in range(attempts): + try: + with urllib.request.urlopen(f"{url}/_debug/statusz", timeout=3): + return + except Exception: + time.sleep(interval) + raise RuntimeError(f"Host Orchestrator at {url} did not become ready") + + +def listening_ports() -> set[int]: + # Opening an HCI connection can create a simulator peer. Inspect the shared + # Pod network namespace instead of disturbing active Bluetooth sessions. + listeners = set() + for table in ("/proc/net/tcp", "/proc/net/tcp6"): + for line in Path(table).read_text().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A": + listeners.add(int(fields[1].rsplit(":", 1)[1], 16)) + return listeners + + +if __name__ == "__main__": + try: + if sys.argv[1] == "--run-exporter": + initialize(sys.argv[2], sys.argv[3], sys.argv[4]) + os.execvp("jmp", ["jmp", "run", "--exporter-config", sys.argv[5]]) + elif sys.argv[1] == "--wait": + wait_ready(sys.argv[2]) + else: + check(sys.argv[1]) + except Exception as exc: + print(f"Cuttlefish unhealthy: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py new file mode 100644 index 000000000..a5a55a9a3 --- /dev/null +++ b/python/packages/jumpstarter-driver-cuttlefish/jumpstarter_driver_cuttlefish/health_test.py @@ -0,0 +1,112 @@ +import io +import json +import time +from pathlib import Path +from unittest.mock import patch + +import pytest + +from .health import check, listening_ports + + +@pytest.fixture +def health_state(tmp_path): + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("runtime-1") + return tmp_path / "health.json", { + "runtime_id_path": str(runtime_id), "runtime_id": "runtime-1", + "url": "http://127.0.0.1:2081", "state": "running", + "group": "cvd", "name": "1", "ports": [7681, 7300], + } + + +def run_check(health_state, cvds=None, ports=None): + path, state = health_state + path.write_text(json.dumps(state)) + if cvds is None: + cvds = [{"group": "cvd", "name": "1", "status": "Running"}] + inventory = json.dumps({"cvds": cvds}).encode() + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", + side_effect=lambda *args, **kwargs: io.BytesIO(inventory)), \ + patch("jumpstarter_driver_cuttlefish.health.listening_ports", + return_value=set(state["ports"] if ports is None else ports)): + check(str(path)) + + +def test_running_guest_and_simulators(health_state): + run_check(health_state) + + +@pytest.mark.parametrize("cvds", [[], [{"group": "cvd", "name": "1", "status": "Stopped"}], + [{"group": "other", "name": "1", "status": "Running"}]]) +def test_guest_failure(health_state, cvds): + with pytest.raises(RuntimeError): + run_check(health_state, cvds=cvds) + + +@pytest.mark.parametrize("missing", [7681, 7300]) +def test_simulator_listener_failure(health_state, missing): + ports = set(health_state[1]["ports"]) - {missing} + with pytest.raises(RuntimeError, match="listener is missing"): + run_check(health_state, ports=ports) + + +def test_intentional_power_off(health_state): + health_state[1]["state"] = "off" + run_check(health_state, cvds=[], ports=[]) + + +def test_bounded_transition(health_state): + health_state[1].update(state="transition", deadline=time.monotonic() + 60) + run_check(health_state, cvds=[], ports=[]) + health_state[1]["deadline"] = time.monotonic() - 1 + with pytest.raises(RuntimeError, match="timed out"): + run_check(health_state) + + +def test_operation_failure(health_state): + health_state[1]["state"] = "failed" + with pytest.raises(RuntimeError, match="failed"): + run_check(health_state) + + +@pytest.mark.parametrize("state", ["off", "running", "transition"]) +def test_runtime_restart_fails_even_when_api_returns(health_state, state): + health_state[1]["state"] = state + Path(health_state[1]["runtime_id_path"]).write_text("runtime-2") + with pytest.raises(RuntimeError, match="runtime restarted"): + run_check(health_state) + + +def test_listener_state_and_ipv6(): + tcp = "header\n0: 0100007F:1E01 00000000:0000 0A\n1: 0100007F:1C84 00000000:0000 01\n" + tcp6 = "header\n0: 00000000000000000000000000000000:45B1 00000000:0000 0A\n" + with patch.object(Path, "read_text", side_effect=[tcp, tcp6]): + assert listening_ports() == {7681, 17841} + + +def test_warm_exporter_before_first_lease(tmp_path): + from .health import initialize + + runtime_id = tmp_path / "runtime-id" + runtime_id.write_text("runtime-1") + state_path = tmp_path / "health.json" + initialize(str(state_path), str(runtime_id), "http://127.0.0.1:2081") + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", return_value=io.BytesIO()): + check(str(state_path)) + runtime_id.write_text("runtime-2") + with pytest.raises(RuntimeError, match="runtime restarted"): + check(str(state_path)) + + +def test_wait_ready_gate(): + from .health import wait_ready + + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", return_value=io.BytesIO()) as urlopen: + wait_ready("http://127.0.0.1:2081", attempts=1, interval=0) + assert urlopen.call_args.args[0] == "http://127.0.0.1:2081/_debug/statusz" + with patch("jumpstarter_driver_cuttlefish.health.urllib.request.urlopen", side_effect=OSError("refused")), \ + patch("jumpstarter_driver_cuttlefish.health.time.sleep") as sleep: + with pytest.raises(RuntimeError, match="did not become ready"): + wait_ready("http://127.0.0.1:2081", attempts=3, interval=5) + assert sleep.call_count == 3