From b4cfc329f0131a01f7d7d39bbd80b62f3cb2df38 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Tue, 4 Aug 2026 02:32:19 -0400 Subject: [PATCH] =?UTF-8?q?feat(edge):=20k3s=20edge-worker=20+=20evolvable?= =?UTF-8?q?=20topology=20=E2=80=94=20Giant=20Swarm,=20reversed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user's model: Giant Swarm runs a cloud management cluster that provisions workload clusters top-down; we REVERSE it — the sovereign edge agent-machine (lightweight single-master k3s) registers UP into a cloud pool as a worker, the twin a rendezvous not a master. And the topology is EVOLVABLE: the same workload climbs k3s-edge -> k3s-server -> k8s-cloud; it shouldn't matter which rung. compute_plane: `k3s-edge` is now a first-class backend — trusted, single-master, residency:edge (the right-sized weight for an ephemeral dev node, not a lesser k8s). Sensitive workloads may run on the sovereign edge. tools/edge_worker.py: - register_worker: the agent-machine joins a cloud pool as a worker; emits a mesh heartbeat so its CPU is real placeable capacity, and advertises its local TopoLVM flash as a shared cluster mount (compute AND storage into the pool). - shared_storage_mount: the local TopoLVM inception mount, cluster-accessible via a container mount. - evolve: migrate a workload across the ladder; state follows via the TopoLVM inception mount; no rung is special. docs/EDGE_WORKER.md articulates the reversed-Giant-Swarm model + why it beats both (sovereign edge AND federate-up). Tests: +5 = 182 tools tests green. compute-plane CapD linked. --- Makefile | 3 ++ capd/compute-plane.mesh.capd.json | 20 +++++++-- docs/EDGE_WORKER.md | 44 +++++++++++++++++++ tools/compute_plane.py | 5 +++ tools/edge_worker.py | 70 +++++++++++++++++++++++++++++++ tools/test_edge_worker.py | 50 ++++++++++++++++++++++ tools/validate.py | 1 + 7 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 docs/EDGE_WORKER.md create mode 100644 tools/edge_worker.py create mode 100644 tools/test_edge_worker.py diff --git a/Makefile b/Makefile index c29325e..fa1d497 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ sphere: ## data-sphere demo: immutable dm-verity sphere, construction-tenancy, i push: ## git-push deploy flow demo: build -> deploy -> per-branch preview cd tools && python3 deploy_flow.py +edge: ## edge-worker demo: register the agent-machine into a cloud pool + evolve topology (reversed Giant Swarm) + cd tools && python3 edge_worker.py + login: ## login/session demo: authenticate the front door (fail-closed) cd tools && python3 login.py diff --git a/capd/compute-plane.mesh.capd.json b/capd/compute-plane.mesh.capd.json index 5f6586c..9d43ce5 100644 --- a/capd/compute-plane.mesh.capd.json +++ b/capd/compute-plane.mesh.capd.json @@ -2,7 +2,7 @@ "capability_id": "caps.compute.mesh-plane@0.1.0", "kind": "compute.placement-plane", "status": "experimental", - "name": "Compute plane — one governed door to any substrate", + "name": "Compute plane \u2014 one governed door to any substrate", "description": "A user develops on a low-mem box and the same workload scales out, seamlessly, over whatever the mesh offers: a k8s service, an HPC/SLURM supercomputer, WASM at the edge, a p2p/hyperswarm mesh, volunteer compute (BOINC / Folding@home / open-HEP-style), or an RLC-style blockchain compute market. The substrate is chosen by the placement broker from per-project/per-account policy and live mesh availability, scaling out where it can and where volunteer compute is offered. Governed and fail-closed: sensitive workloads never land on an untrusted (volunteer/p2p/blockchain) backend, every placement is sealed, and with no allowed+available backend it falls back to local or blocks rather than shipping work somewhere the policy forbids. Configured in the portal dashboard.", "links": { "broker": "tools/compute_plane.py", @@ -13,7 +13,9 @@ "grant_authority": "tools/mcp_a2a_grant.py", "portal": "tools/portal_server.py", "agent_surface": "tools/mcp_ops_server.py", - "reference_pattern": "BOINC computing-preferences + IBM Parallel Environment (HPC toolkit) + Docker volume drivers — met and bettered: one governed plane across all substrates, not per-substrate config silos" + "reference_pattern": "BOINC computing-preferences + IBM Parallel Environment (HPC toolkit) + Docker volume drivers \u2014 met and bettered: one governed plane across all substrates, not per-substrate config silos", + "edge_worker": "tools/edge_worker.py (register agent-machine into a cloud pool; evolvable topology)", + "topology_witness": "docs/EDGE_WORKER.md" }, "composes_with": { "inner_loop": "caps.dev.devspace-inner-loop@0.1.0", @@ -27,7 +29,17 @@ "sensitive_never_untrusted": true, "per_project_config": true, "per_account_config": true, - "read_only_portal": true + "read_only_portal": true, + "k3s_edge_single_master": true, + "evolvable_topology": true }, - "backends": ["local", "k8s", "hpc-slurm", "wasm-edge", "p2p-mesh", "volunteer-boinc", "blockchain-rlc"] + "backends": [ + "local", + "k8s", + "hpc-slurm", + "wasm-edge", + "p2p-mesh", + "volunteer-boinc", + "blockchain-rlc" + ] } diff --git a/docs/EDGE_WORKER.md b/docs/EDGE_WORKER.md new file mode 100644 index 0000000..89330e5 --- /dev/null +++ b/docs/EDGE_WORKER.md @@ -0,0 +1,44 @@ +# Edge-worker + evolvable topology — Giant Swarm, reversed + +## The model + +**Giant Swarm** runs a cloud **management cluster** that provisions and operates **workload clusters** +top-down (k8s-on-k8s, Cluster API). Powerful, but cloud-owns-edge, and heavy. + +We **reverse it.** The edge **agent-machine** — a lightweight, **single-master k3s** (k3s-in-docker on +the M2, or k3s on a server) — is sovereign and local-first, and it **registers *up* into a cloud pool +as a worker.** The cloud **twin** is a rendezvous, not a master. Two design facts drive this: + +- **An ephemeral dev node doesn't need HA.** Single-master k3s is the right weight at the edge; + redundant multi-master k8s is for the cloud. So `k3s-edge` is a distinct, trusted backend from full + `k8s` — not a lesser one, the *right-sized* one. +- **The topology is evolvable, not pinned.** The same workload climbs the ladder as needs grow, and + it shouldn't matter which rung it's on: + + ``` + k3s-edge ──► k3s-server ──► k8s-cloud + (in-docker (a box on (redundant, + on the box) the LAN/DC) multi-master) + ``` + +## What's built (`tools/edge_worker.py`) + +- **`register_worker`** — the agent-machine joins a cloud **pool** as a worker (the reversed + direction). It emits a **mesh heartbeat** (`mesh_telemetry`), so the edge node's CPU becomes real, + **placeable** capacity the compute plane can schedule onto — the box's compute is now in the pool. +- **`shared_storage_mount`** — the agent-machine's local **TopoLVM flash** exposed to the cluster as a + shared container mount (`topolvm-provisioner`, at `/var/lib/sourceos/inception`). The registered + worker contributes **storage as well as compute**. +- **`evolve`** — migrate a workload across the ladder (`k3s-edge ⇄ k3s-server ⇄ k8s-cloud`). State + follows via the TopoLVM inception mount; no rung is special, migration is just an index move. +- **`k3s-edge` backend** (`compute_plane`) — trusted, single-master, `residency: edge`. A sensitive + workload may run on the sovereign edge; the Needs firewall and placement treat it as first-class. + +## Why this beats both models + +Giant Swarm gives you managed clusters but the cloud owns them. A plain k3s gives you a sovereign +edge but it's an island. The **edge-worker + evolvable topology** gives you both: the edge is +**sovereign and local-first** (your k3s, your TopoLVM flash, your data), *and* it **federates up** — +registering compute + storage into a cloud pool when you want scale, migrating workloads up the +ladder as needs grow, and falling back to fully-local single-master when you don't. Same governed +placement/grant/needs plane across every rung. It shouldn't matter where it runs — and now it doesn't. diff --git a/tools/compute_plane.py b/tools/compute_plane.py index 4bc409a..dcfcfba 100644 --- a/tools/compute_plane.py +++ b/tools/compute_plane.py @@ -25,6 +25,10 @@ # a coarse scale-out capacity rank (1 = a single laptop, 10 = a global volunteer grid). BACKENDS = { "local": {"kind": "local", "trust": "trusted", "elasticity": 1, "gpu": False}, + # k3s-edge: the sovereign agent-machine — lightweight SINGLE-MASTER k3s (in-docker on the box or + # on a server). No HA redundancy (an ephemeral dev node doesn't need it); registers UP into a + # cloud pool as a worker (Giant Swarm, reversed) and its workloads evolve up to full k8s. + "k3s-edge": {"kind": "k3s-lightweight", "trust": "trusted", "elasticity": 3, "gpu": False}, "k8s": {"kind": "container", "trust": "trusted", "elasticity": 6, "gpu": True}, "hpc-slurm": {"kind": "hpc", "trust": "trusted", "elasticity": 9, "gpu": True}, "wasm-edge": {"kind": "wasm", "trust": "trusted", "elasticity": 7, "gpu": False}, @@ -40,6 +44,7 @@ # only be satisfied by a backend that provably provides it — a soft Want never masquerades as a Need. BACKEND_CAPS = { "local": {"residency": "local", "no_egress": True}, + "k3s-edge": {"residency": "edge", "single_master": True}, "k8s": {"residency": "cluster", "fips": True}, "hpc-slurm": {"residency": "cluster", "fips": True, "tee": True}, "wasm-edge": {"residency": "edge", "deterministic": True}, diff --git a/tools/edge_worker.py b/tools/edge_worker.py new file mode 100644 index 0000000..22c2ea1 --- /dev/null +++ b/tools/edge_worker.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Edge-worker registration + evolvable topology — Giant Swarm, reversed. + +Giant Swarm runs a cloud MANAGEMENT cluster that provisions workload clusters top-down (k8s-on-k8s / +Cluster API). We invert it: the edge agent-machine — a lightweight, SINGLE-MASTER **k3s** (k3s-in- +docker on the box, or k3s on a server) — is sovereign and local-first, and REGISTERS UP into a cloud +pool as a **worker**. The cloud twin is a rendezvous, not a master. + +Two consequences the user called out: + * an ephemeral dev node doesn't need HA redundancy, so single-master k3s is right at the edge; the + topology is EVOLVABLE — the same workload climbs `k3s-edge -> k3s-server -> k8s-cloud` as needs + grow, and it shouldn't matter which rung it's on. + * the agent-machine's local **TopoLVM flash** is exposed to the cluster as a shared container mount, + so the registered worker contributes both compute AND storage to the pool. +""" +from __future__ import annotations + +import mesh_telemetry as mt + +# The evolvable topology ladder. A workload/node climbs it as needs grow; migration up or down is +# allowed — the topology is not pinned to any rung. +TOPOLOGY_LADDER = ["k3s-edge", "k3s-server", "k8s-cloud"] + + +def shared_storage_mount(*, node_id: str, pvc: str = "inception-mount", + path: str = "/var/lib/sourceos/inception") -> dict: + """The agent-machine's local TopoLVM flash, exposed to the cluster as a shared container mount.""" + return {"pvc": pvc, "path": path, "storage_class": "topolvm-provisioner", "node": node_id, + "access": "ReadWriteOnce", "shared_via": "container mount on the registered worker node"} + + +def register_worker(*, node_id: str, pool: str, cpu: int, mem_gb: int, storage_gb: int, + gpu: bool = False, distro: str = "k3s-edge", heartbeats_dir=None) -> dict: + """The agent-machine joins a cloud POOL as a worker (Giant Swarm, reversed). Emits a mesh heartbeat + so placement sees its capacity, and advertises its TopoLVM storage. Returns the registration.""" + backend = "k3s-edge" if distro.startswith("k3s") else "k8s" + rec = {"node_id": node_id, "pool": pool, "distro": distro, "backend": backend, "role": "worker", + "cpu": cpu, "mem_gb": mem_gb, "storage_gb": storage_gb, "gpu": gpu, "registered": True, + "shared_storage": shared_storage_mount(node_id=node_id)} + if heartbeats_dir is not None: + mt.write_heartbeat(heartbeats_dir, node_id, backend, cpu) # now visible to compute_plane.place() + return rec + + +def evolve(*, workload: dict, from_backend: str, to_backend: str) -> dict: + """Migrate a workload across the topology ladder (evolvable topology). Data follows via the + inception mount; the direction is just an index move on the ladder — no rung is special.""" + if from_backend not in TOPOLOGY_LADDER or to_backend not in TOPOLOGY_LADDER: + return {"ok": False, "reason": f"backend not on the topology ladder {TOPOLOGY_LADDER}"} + i, j = TOPOLOGY_LADDER.index(from_backend), TOPOLOGY_LADDER.index(to_backend) + return {"ok": True, "workload": workload.get("name"), "from": from_backend, "to": to_backend, + "direction": "up" if j > i else "down" if j < i else "same", + "carries_inception_mount": bool(workload.get("inception_pvc")), + "note": f"migrate {from_backend} -> {to_backend}; state follows via the TopoLVM inception mount"} + + +if __name__ == "__main__": + import json + import tempfile + with tempfile.TemporaryDirectory() as td: + reg = register_worker(node_id="m2-agent", pool="cloud-pool-a", cpu=8, mem_gb=16, + storage_gb=200, gpu=False, distro="k3s-edge", heartbeats_dir=td) + avail = mt.MeshRegistry.from_dir(td).availability() + wl = {"name": "trainer", "inception_pvc": "inception-mount"} + print(json.dumps({ + "registered": {"node": reg["node_id"], "pool": reg["pool"], "backend": reg["backend"], + "storage": reg["shared_storage"]["storage_class"]}, + "mesh_sees": avail, + "evolve": evolve(workload=wl, from_backend="k3s-edge", to_backend="k8s-cloud"), + }, indent=2)) diff --git a/tools/test_edge_worker.py b/tools/test_edge_worker.py new file mode 100644 index 0000000..9faa3f1 --- /dev/null +++ b/tools/test_edge_worker.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Tests for edge-worker registration + evolvable topology (Giant Swarm, reversed).""" +import tempfile + +import compute_plane as cp +import edge_worker as ew +import mesh_telemetry as mt + + +def test_register_worker_joins_a_pool_and_becomes_visible_to_placement(): + with tempfile.TemporaryDirectory() as td: + reg = ew.register_worker(node_id="m2", pool="cloud-pool-a", cpu=8, mem_gb=16, + storage_gb=200, distro="k3s-edge", heartbeats_dir=td) + assert reg["backend"] == "k3s-edge" and reg["role"] == "worker" and reg["registered"] + # the edge node's capacity is now real, placeable mesh availability (registered UP into the pool) + assert mt.MeshRegistry.from_dir(td).availability().get("k3s-edge") == 8 + + +def test_shared_storage_is_the_topolvm_inception_mount(): + m = ew.shared_storage_mount(node_id="m2") + assert m["storage_class"] == "topolvm-provisioner" and m["path"] == "/var/lib/sourceos/inception" + assert "container mount" in m["shared_via"] + + +def test_evolve_migrates_up_and_down_the_ladder_carrying_state(): + wl = {"name": "svc", "inception_pvc": "inception-mount"} + up = ew.evolve(workload=wl, from_backend="k3s-edge", to_backend="k8s-cloud") + assert up["ok"] and up["direction"] == "up" and up["carries_inception_mount"] is True + down = ew.evolve(workload=wl, from_backend="k8s-cloud", to_backend="k3s-edge") + assert down["direction"] == "down" + + +def test_evolve_rejects_a_backend_off_the_ladder(): + assert ew.evolve(workload={}, from_backend="k3s-edge", to_backend="volunteer-boinc")["ok"] is False + + +def test_k3s_edge_is_trusted_and_placeable(): + # the sovereign edge is a trusted backend — a sensitive workload may run there. + d = cp.place({"sensitivity": "sensitive", "scalable": False}, + {"allowed_backends": ["k3s-edge"]}, {"k3s-edge": 8}) + assert d["backend"] == "k3s-edge" and d["backend_trust"] == "trusted" + + +if __name__ == "__main__": + import sys + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + for fn in fns: + fn() + print(f"ok: {len(fns)} edge-worker tests passed") + sys.exit(0) diff --git a/tools/validate.py b/tools/validate.py index f51e795..2f187e6 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -50,6 +50,7 @@ "tools/provisioning.py", "tools/deploy_flow.py", "tools/login.py", + "tools/edge_worker.py", ] CAPD_KEYS = ("capability_id", "kind", "status", "links", "composes_with", "policy") # Every CapD in capd/ must carry the core keys and parse — not just the flagship control-plane one.