feat: add cuttlefish provisioner for dynamic exporters

controller/cmd/exporter-set-controller/main.go

@@ -36,6 +36,7 @@ import (
 	jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1"
 	virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1"
 	"github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset"
+	"github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish"
 	"github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/qemu"
 )
 
@@ -159,9 +160,14 @@ func main() {
 // Add new provisioners here as they are implemented.
 func selectProvisioner(name string) (exporterset.Provisioner, error) {
 	switch name {
+	case cuttlefish.ProvisionerName:
+		return cuttlefish.New(version), nil
 	case qemu.ProvisionerName:
 		return qemu.New(version), nil
 	default:
-		return nil, fmt.Errorf("unknown provisioner %q; supported: %s", name, qemu.ProvisionerName)
+		return nil, fmt.Errorf(
+			"unknown provisioner %q; supported: %s, %s",
+			name, qemu.ProvisionerName, cuttlefish.ProvisionerName,
+		)
 	}
 }

controller/deploy/operator/config/rbac/role.yaml

@@ -171,6 +171,17 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - operator.jumpstarter.dev
   resources:

controller/deploy/operator/internal/controller/jumpstarter/exporterset.go

@@ -517,6 +517,11 @@ func exporterSetPolicyRules() []rbacv1.PolicyRule {
 			Resources: []string{"leases"},
 			Verbs:     []string{"get", "list", "watch"},
 		},
+		{
+			APIGroups: []string{"networking.k8s.io"},
+			Resources: []string{"networkpolicies"},
+			Verbs:     []string{"get", "list", "watch", "create", "update", "patch"},
+		},
 		{
 			APIGroups: []string{""},
 			Resources: []string{"pods"},

controller/deploy/operator/internal/controller/jumpstarter/exporterset_test.go

@@ -120,6 +120,16 @@ var _ = Describe("exporterSetPolicyRules", func() {
 		Expect(groups).To(HaveKey("coordination.k8s.io"))
 	})
 
+	It("should reconcile runtime network isolation policies", func() {
+		for _, rule := range rules {
+			if containsString(rule.APIGroups, "networking.k8s.io") && containsString(rule.Resources, "networkpolicies") {
+				Expect(rule.Verbs).To(ContainElements("get", "list", "watch", "create", "update", "patch"))
+				return
+			}
+		}
+		Fail("no rule found for runtime network policies")
+	})
+
 	It("should grant read-only access on exportersets (no create/update/delete)", func() {
 		for _, rule := range rules {
 			if containsString(rule.APIGroups, "virtualtarget.jumpstarter.dev") &&

controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go

@@ -90,6 +90,7 @@ type JumpstarterReconciler struct {
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
 
 // Networking resources
+// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete

controller/internal/controller/lease_controller_test.go

@@ -1492,7 +1492,7 @@ var _ = Describe("Scheduled Leases", func() {
 	When("creating lease with BeginTime + Duration (scheduled lease)", func() {
 		It("should wait until BeginTime before acquiring exporter", func() {
 			lease := leaseDutA2Sec.DeepCopy()
-			futureTime := metav1.NewTime(time.Now().Truncate(time.Second).Add(1 * time.Second))
+			futureTime := metav1.NewTime(time.Now().Add(2 * time.Second).Truncate(time.Second))
 			lease.Spec.BeginTime = &futureTime
 			lease.Spec.Duration = &metav1.Duration{Duration: 1 * time.Second}
 			lease.Spec.EndTime = nil

controller/internal/exporterset/networkpolicy_test.go

@@ -0,0 +1,94 @@
+package exporterset
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1"
+	"github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/provisioners/cuttlefish"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+)
+
+func TestCuttlefishNetworkPolicyReconciliation(t *testing.T) {
+	scheme := newScheme(t)
+	if err := networkingv1.AddToScheme(scheme); err != nil {
+		t.Fatal(err)
+	}
+	es := makeExporterSet()
+	c := fake.NewClientBuilder().WithScheme(scheme).Build()
+	r := &ExporterSetReconciler{Client: c, Scheme: scheme, Provisioner: cuttlefish.New("dev")}
+	ctx := context.Background()
+	if err := r.syncNetworkPolicy(ctx, es); err != nil {
+		t.Fatal(err)
+	}
+	policy := &networkingv1.NetworkPolicy{}
+	key := client.ObjectKey{Namespace: es.Namespace, Name: "cuttlefish-" + string(es.UID)}
+	if err := c.Get(ctx, key, policy); err != nil {
+		t.Fatal(err)
+	}
+	if !metav1.IsControlledBy(policy, es) || len(policy.Spec.Ingress) != 0 {
+		t.Fatalf("unowned or permissive policy: %#v", policy)
+	}
+	policy.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{{}}
+	if err := c.Update(ctx, policy); err != nil {
+		t.Fatal(err)
+	}
+	if err := r.syncNetworkPolicy(ctx, es); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Get(ctx, key, policy); err != nil {
+		t.Fatal(err)
+	}
+	if len(policy.Spec.Ingress) != 0 {
+		t.Fatal("policy drift not corrected")
+	}
+	if err := c.Delete(ctx, policy); err != nil {
+		t.Fatal(err)
+	}
+	if err := r.syncNetworkPolicy(ctx, es); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Get(ctx, key, policy); err != nil {
+		t.Fatal("deleted policy not recreated", err)
+	}
+}
+
+func TestPolicyFailurePreventsWorkloadCreation(t *testing.T) {
+	scheme := newScheme(t)
+	if err := networkingv1.AddToScheme(scheme); err != nil {
+		t.Fatal(err)
+	}
+	es := makeExporterSet()
+	vtc := &virtualtargetv1alpha1.VirtualTargetClass{
+		ObjectMeta: metav1.ObjectMeta{Name: es.Spec.VirtualTargetClassName, Namespace: es.Namespace},
+		Spec:       virtualtargetv1alpha1.VirtualTargetClassSpec{Provisioner: cuttlefish.ProvisionerName},
+	}
+	c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(es, vtc).WithInterceptorFuncs(interceptor.Funcs{
+		Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
+			if _, ok := obj.(*networkingv1.NetworkPolicy); ok {
+				return errors.New("network policy denied")
+			}
+			return c.Create(ctx, obj, opts...)
+		},
+	}).Build()
+	r := &ExporterSetReconciler{Client: c, Scheme: scheme, Provisioner: cuttlefish.New("dev")}
+	_, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(es)})
+	if err == nil || !strings.Contains(err.Error(), "network policy denied") {
+		t.Fatalf("expected policy error: %v", err)
+	}
+	var pods corev1.PodList
+	if err := c.List(context.Background(), &pods); err != nil {
+		t.Fatal(err)
+	}
+	if len(pods.Items) != 0 {
+		t.Fatal("created workload without network isolation")
+	}
+}

controller/internal/exporterset/provisioners/cuttlefish/README.md

@@ -0,0 +1,197 @@
+# Cuttlefish ExporterSets
+
+Each exporter owns one CVD. The managed backend uses Host Orchestrator over
+HTTP inside the Pod, crosvm with private userspace VSOCK, and netsim Bluetooth.
+The standalone Python driver continues to support externally managed HTTP hosts.
+An exec backend is a separate follow-up.
+
+## Workload admission and networking
+
+Create a dedicated service account in the ExporterSet namespace and set
+`parameters.service_account_name` to its name. `default` is rejected. The Pod
+does not mount a Kubernetes API token. `runtime_privileged: true` is required;
+`false` is rejected until device permissions and capabilities are supported.
+
+On Kubernetes, create the namespace and workload account. If Pod Security
+Admission is enabled, the namespace must permit privileged workloads; the
+`baseline` and `restricted` profiles reject this Pod. For a dedicated test
+namespace:
+
+```sh
+kubectl create namespace cuttlefish-lab
+kubectl label namespace cuttlefish-lab pod-security.kubernetes.io/enforce=privileged
+kubectl -n cuttlefish-lab create serviceaccount cuttlefish-runtime
+```
+
+This namespace setting permits privileged workloads for every account in that
+namespace, so restrict who can create workloads there. Other admission policies
+must also allow the Pod's privileged containers, hostPath devices and container
+UIDs. The workload account needs no Kubernetes API permissions. Nodes must expose
+the required KVM and networking devices; VM-based nodes need nested virtualization.
+The cluster must support native sidecar containers: the runtime init containers
+use `restartPolicy: Always`.
+See [Kubernetes Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/).
+
+On OpenShift, a cluster administrator must grant that workload account access
+to an SCC permitting privileged containers, hostPath devices, and the UIDs used
+by **all** containers, including image copy/fetch and permission init containers.
+For initial testing, a scoped grant to the built-in privileged SCC is:
+
+```sh
+oc -n cuttlefish-lab create serviceaccount cuttlefish-runtime
+oc adm policy add-scc-to-user privileged -z cuttlefish-runtime -n cuttlefish-lab
+```
+
+This grant is for the workload account, not the ExporterSet controller. Merely
+setting `runtime_privileged` cannot grant SCC admission. Check the admitted Pod's
+`openshift.io/scc` annotation, completion of init containers, and an actual guest
+boot. A server-side dry run only establishes admission, not device access.
+See [OpenShift SCC documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/authentication_and_authorization/managing-pod-security-policies).
+
+The controller creates and reconciles an ExporterSet-owned NetworkPolicy before
+creating workloads. It denies Pod ingress, including Host Orchestrator, nginx,
+ADB and simulator listeners, while leaving outbound exporter connections and
+same-Pod loopback traffic available. Deploy with a CNI that enforces NetworkPolicy.
+Other policies must not grant ingress to these Pods: Kubernetes allow rules are
+additive. Verify denial from a second Pod, using the actual runtime image;
+loopback addresses in driver configuration do not change the server's listeners.
+NetworkPolicy does not isolate privileged containers from their node.
+
+Controllers deployed outside the operator also need `get,list,watch,create,update,patch`
+on `networking.k8s.io/networkpolicies`. The operator supplies these permissions.
+Existing Pods must be drained and replaced to receive the isolation label,
+service account, runtime marker and managed driver configuration.
+
+## Configuration and resource allocation
+
+The provisioner requires exactly one Cuttlefish driver and one
+`env_config.instances` entry. It fixes the managed endpoint to
+`http://127.0.0.1:2081` and `instance_num` to 1. A different
+`host_orchestrator_port` is rejected because the upstream image fixes its service
+and nginx upstream configuration to 2081.
+
+Guest defaults are 4 CPUs and 8192 MiB. Runtime memory requests default to the
+**effective** guest memory plus `runtime_memory_overhead_mb` (2048 MiB by default).
+Explicit requests and limits must cover that budget. Increase the overhead for
+larger simulator workloads. Guest values in driver `env_config` take precedence
+over `vm_cpus` and `vm_memory_mb` when calculating the budget. CPU requests default
+to guest CPUs, or to an explicit CPU limit; CPU overcommit remains configurable.
+
+Relay ports must be distinct integers in 1..65535 and must not collide with the
+managed runtime's service ports. Defaults are 17681 (netsim) and 17300 (HCI).
+The relay supervisor exits if either listener process dies.
+
+`create_cvd()` in managed mode accepts only the exact configured `env_config`,
+checks the entire Host Orchestrator inventory, and serializes creation with other
+lifecycle operations. Destroy the existing CVD before creating another. This
+prevents alternate API payloads or concurrent calls from exceeding the configured
+instance count and memory budget. Arbitrary template code and driver imports are
+administrator-controlled; they are not a security boundary against a malicious
+cluster administrator.
+
+## VSOCK and Bluetooth compatibility
+
+Managed configuration sets:
+
+```yaml
+env_config:
+  netsim_bt: true
+  instances:
+    - vm:
+        crosvm:
+          vhost_user_vsock: "true"
+```
+
+The string value is required by the upstream configuration schema. The runtime
+and guest must support that backend. The provisioner does not mount
+`/dev/vhost-vsock`; each Pod has private runtime files and Unix sockets. A
+privileged runtime still has broad node access, so this is not containment of a
+compromised runtime. QEMU, gem5, disabled userspace VSOCK, and standalone RootCanal
+(`netsim_bt: false`) are rejected for this managed backend. The referenced
+upstream standalone RootCanal proxy does not propagate the userspace VSOCK flag.
+
+Before approving a runtime/build pair, boot two Pods on the same node using the
+default identical guest CID, verify their generated configuration and private
+`vhost.socket`/`vm.vsock` paths, and exercise netsim plus Bluetooth peer traffic
+in both leases. Stop one guest and verify the other remains usable. Source/unit
+tests cannot establish compatibility of a mutable image tag or guest build.
+
+## Image PVCs and reproducibility
+
+A prewarmed image PVC is mounted read-only, then copied into a private writable
+`emptyDir`. It remains a Pod volume after the init container exits. Read-only
+mounting does **not** change the PVC's access mode or remove attachment constraints.
+
+For a pool spanning nodes, use storage supporting ReadOnlyMany or ReadWriteMany
+with the required topology. For ReadWriteOnce, keep readers on one compatible
+node; ReadWriteOncePod permits only one Pod. Alternatively fetch images per Pod.
+See [Kubernetes access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes).
+
+Use an immutable runtime manifest digest, relay digest and Android build ID for
+repeatable replacements. This example is a template: substitute verified values
+from a tested pair; the placeholders are not a published compatibility claim.
+Pin the exporter image built with this provisioner/driver change too.
+
+```yaml
+apiVersion: virtualtarget.jumpstarter.dev/v1alpha1
+kind: VirtualTargetClass
+metadata:
+  name: cuttlefish
+  namespace: cuttlefish-lab
+spec:
+  provisioner: cuttlefish.jumpstarter.dev
+  parameters:
+    service_account_name: cuttlefish-runtime
+    runtime_privileged: true
+    fetch_images: true
+    default_build: "<android-build-id>/aosp_cf_x86_64_auto-userdebug"
+    relay_image: "docker.io/alpine/socat@sha256:<relay-manifest-digest>"
+    vm_cpus: 4
+    vm_memory_mb: 8192
+    runtime_memory_overhead_mb: 2048
+  images:
+    runtime:
+      image: "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orchestration/cuttlefish-orchestration@sha256:<runtime-manifest-digest>"
+    exporter:
+      image: "quay.io/jumpstarter-dev/jumpstarter@sha256:<exporter-manifest-digest>"
+  scheduling:
+    resources:
+      requests:
+        cpu: "4"
+        memory: 10Gi
+      limits:
+        memory: 10Gi
+```
+
+Record both the resolved runtime image digest and the guest `fetcher_config.json`
+with validation results. A prewarmed PVC needs the same build provenance.
+
+## Failure and recovery behavior
+
+The exporter liveness probe reads state written atomically by the managed driver.
+It always checks Host Orchestrator availability and a per-start runtime ID.
+When the guest is expected to run, it also checks the CVD inventory/status and
+simulator/relay TCP listeners in the shared network namespace. It inspects
+listeners instead of opening HCI connections that could disturb Bluetooth peers.
+A listening socket alone does not prove simulator protocol correctness.
+
+Intentional stop, destroy and reset leave the Pod healthy without a guest.
+Create/start/restart/powerwash receive bounded transition time; failed or expired
+operations fail health checks. A runtime sidecar restart invalidates the exporter
+even if its HTTP API comes back: it must not silently resume a lease after losing
+runtime state. Six failed checks, ten seconds apart, terminate the exporter.
+
+With `ExitAndReplace`, the failed exporter remains associated with an active
+lease. Release that lease to let the controller delete and replace the Pod;
+reacquire a lease for a fresh device. Automatic replacement during an active
+lease is not attempted. Use `ExitAndReplace` for managed Cuttlefish pools.
+
+Validate recovery on disposable leased Pods by terminating the VMM, netsim, each
+relay, and then the runtime sidecar in separate trials. A component may recover
+within the probe failure window; verify guest and simulator functionality after
+recovery. For unrecovered failures and runtime sidecar restarts, confirm liveness
+failure, client disconnect, retention while leased, and replacement after release. Also
+exercise intentional power off/on and destroy/create, which must remain healthy.
+Actual CNI enforcement and colocated VSOCK/BT trials are required integration
+checks beyond the local regression tests. OpenShift deployments additionally
+require SCC admission and guest boot validation.