From c68b5a4fc824ec7209d89dad34877519f433c6fe Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Tue, 4 Aug 2026 02:02:01 -0400 Subject: [PATCH] feat(inference+mobile): sovereign inference on our mesh + installable mobile PWA (twin or box) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two asks, one capstone: run LLMs on OUR infrastructure (not a cloud provider), and see/reach it from a phone. Sovereign inference (tools/inference.py): - a model is an immutable DATA SPHERE (provenance, pinned integrity, residency-fenced); loading weights needs a read Grant; a mutated model is un-citable. - inference_service_workload: serving a model is a GPU workload the compute plane places on a TRUSTED backend (Needs firewall keeps a sensitive model off untrusted/volunteer/vendor). - route_inference: FAIL-CLOSED sovereign-first. A sensitive prompt (or a residency-fenced model) goes to a sovereign endpoint or BLOCKS — it never leaves for a cloud LLM. Non-sensitive may fall back to a vendor connector only when policy allows. This is the difference between "our infrastructure" and "a cloud provider like Claude." Mobile PWA (portal_server.py): - installable (manifest.webmanifest + service worker, offline-ish shell), mobile-responsive, apple web-app tags; a twin/box endpoint badge (SOURCEOS_ENDPOINT). - /api/inference surfaces the sovereign posture + per-model routing; a "Sovereign inference" console section shows sensitive prompts routing sovereign or blocked. Access model: default to the TWIN (always-on rendezvous — the box sleeps, the twin doesn't; grants + coordinator live there), opt-in DIRECT to the box for LAN/offline. Same reference-vs-direct lattice from the mount analysis, applied to control access. capd/sovereign-inference.mesh.capd.json. Tests: +7 inference +4 portal = 156 tools tests green. --- Makefile | 3 + capd/sovereign-inference.mesh.capd.json | 30 ++++++++++ tools/inference.py | 77 +++++++++++++++++++++++++ tools/portal_server.py | 70 ++++++++++++++++++++-- tools/test_inference.py | 67 +++++++++++++++++++++ tools/test_portal_server.py | 23 ++++++++ tools/validate.py | 3 + 7 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 capd/sovereign-inference.mesh.capd.json create mode 100644 tools/inference.py create mode 100644 tools/test_inference.py diff --git a/Makefile b/Makefile index 58f586e..c996675 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ lease: ## pull/lease scheduler demo: workers pull WUs, crash-stop re-lending, or sphere: ## data-sphere demo: immutable dm-verity sphere, construction-tenancy, intent x link x durability cd tools && python3 data_sphere.py +inference: ## sovereign inference demo: models as data spheres, fail-closed sovereign routing + cd tools && python3 inference.py + availability: ## report the estate's availability-maturity grades (the Zero-Downtime legend) cd tools && python3 availability.py diff --git a/capd/sovereign-inference.mesh.capd.json b/capd/sovereign-inference.mesh.capd.json new file mode 100644 index 0000000..32d82c0 --- /dev/null +++ b/capd/sovereign-inference.mesh.capd.json @@ -0,0 +1,30 @@ +{ + "capability_id": "caps.inference.sovereign@0.1.0", + "kind": "inference.sovereign", + "status": "experimental", + "name": "Sovereign inference — run LLMs on our own mesh, not a cloud provider", + "description": "The whole point of a sovereign PaaS: a sensitive prompt must never leave for a vendor LLM (OpenAI/Anthropic/Gemini). The mesh serves its own models — weights are immutable data spheres (provenance-tracked, residency ring-fenced, read-Grant-gated), served on a TRUSTED GPU backend the Needs firewall keeps sensitive work off untrusted/volunteer/vendor nodes. Inference routing is fail-closed: sensitive inference goes to a sovereign endpoint or it BLOCKS; it never silently falls back to a cloud connector. Where it runs — the always-on cloud twin or the box (direct/LAN) — is a placement decision; both are sovereign, twin by default.", + "links": { + "engine": "tools/inference.py", + "models": "tools/data_sphere.py", + "placement": "tools/compute_plane.py", + "grant_authority": "tools/mcp_a2a_grant.py", + "portal": "tools/portal_server.py", + "reference_pattern": "self-hosted vLLM/llama.cpp/Ollama/TGI on our mesh vs cloud LLM APIs — sovereign, governed, sensitive-data-safe; models as immutable data spheres" + }, + "composes_with": { + "data_spheres": "caps.data.spheres@0.1.0", + "compute_plane": "caps.compute.mesh-plane@0.1.0", + "control_plane": "caps.infra.paas.continuum-local@0.1.0", + "scales_up_to": "caps.infra.cluster-scaleup.hyperswarm@0.1.0" + }, + "policy": { + "availability": "needs-work", + "sovereign_first": true, + "sensitive_never_vendor": true, + "models_as_data_spheres": true, + "fail_closed": true, + "gpu_trusted_only": true, + "evidence_emitting": true + } +} diff --git a/tools/inference.py b/tools/inference.py new file mode 100644 index 0000000..2ec9bf8 --- /dev/null +++ b/tools/inference.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Sovereign inference — run LLMs on OUR mesh, not a cloud provider. + +The whole point of a sovereign PaaS: a sensitive prompt must NEVER leave for a vendor LLM +(OpenAI/Anthropic/Gemini/…). The mesh serves its own models — weights are immutable DATA SPHERES +(provenance-tracked, residency ring-fenced), served on a TRUSTED GPU backend behind a Grant — and +inference routing is fail-closed: sensitive inference goes to a sovereign endpoint or it BLOCKS; it +never silently falls back to a cloud connector. That is the difference between "our infrastructure" +and "a cloud provider like Claude." + +Where inference runs — the durable **twin** (always-on cloud K3s) or the **box** (direct/LAN when it +is up) — is a placement decision the compute plane already makes; both are sovereign, and the twin +is the default rendezvous because the box sleeps and the twin does not. +""" +from __future__ import annotations + +import data_sphere as ds + +ENGINES = ("vllm", "llama.cpp", "ollama", "tgi") + + +def model_sphere(*, name: str, version: str, weights_digest: str, params_b: float, + engine: str = "vllm", residency: str = "cluster") -> dict: + """A model is a data sphere: immutable weights, pinned integrity, provenance, residency-fenced. + Loading the weights therefore needs a read Grant, and a mutated model is un-citable.""" + s = ds.mint_sphere(name=f"model/{name}", version=version, + content={"weights": weights_digest, "params_b": params_b, "engine": engine}, + residency=residency, direction="ingress", + provenance={"kind": "model-weights", "params_b": params_b, "engine": engine}) + s["model_name"] = name + s["params_b"] = params_b + s["engine"] = engine + return s + + +def inference_service_workload(model: dict, *, replicas: int = 1, sensitivity: str = "sensitive") -> dict: + """Serving a model = a GPU workload the compute plane places on a TRUSTED backend (the Needs + firewall keeps a sensitive model off untrusted/volunteer/vendor backends). Dispatch it with the + executor like any other workload; reading the weights needs a read Grant on the model sphere.""" + return {"name": "infer-" + model["model_name"].replace("/", "-"), + "kind": "inference-service", "engine": model.get("engine", "vllm"), + "model_sphere": model["sphere_id"], "needs_gpu": True, "scalable": True, + "replicas": replicas, "effect": "compute", "sensitivity": sensitivity, + "needs": {"residency": model.get("residency", "cluster")}} + + +def route_inference(*, model: dict, sovereign_endpoints: list, prompt_sensitivity: str = "sensitive", + allow_vendor: bool = False) -> dict: + """Fail-closed sovereign-first routing. Returns {route, endpoint, reason}. A sensitive prompt (or + a residency-fenced model) is sent to a sovereign endpoint or BLOCKED — never a cloud LLM.""" + if sovereign_endpoints: + return {"route": "sovereign", "endpoint": sovereign_endpoints[0], + "reason": "served on our own mesh — the prompt never leaves"} + sovereign_required = (prompt_sensitivity == "sensitive" + or model.get("residency") in ("local", "cluster", "eu")) + if sovereign_required: + return {"route": "blocked", "endpoint": None, + "reason": "no sovereign endpoint up; REFUSING to send sensitive inference to a cloud LLM"} + if allow_vendor: + return {"route": "vendor", "endpoint": "connector", + "reason": "non-sensitive, no sovereign endpoint: policy-allowed vendor fallback"} + return {"route": "blocked", "endpoint": None, + "reason": "no sovereign endpoint and vendor fallback not permitted"} + + +if __name__ == "__main__": + import json + m = model_sphere(name="llama-3-70b", version="q4", weights_digest="sha256:" + "ab" * 32, + params_b=70, engine="vllm", residency="any") + print(json.dumps({ + "model_sphere": m["sphere_id"], + "service": inference_service_workload(m)["name"], + "sensitive_no_endpoint": route_inference(model=m, sovereign_endpoints=[], prompt_sensitivity="sensitive")["route"], + "sensitive_with_endpoint": route_inference(model=m, sovereign_endpoints=["twin:vllm:8000"])["route"], + "normal_vendor_fallback": route_inference(model=m, sovereign_endpoints=[], + prompt_sensitivity="normal", allow_vendor=True)["route"], + }, indent=2)) diff --git a/tools/portal_server.py b/tools/portal_server.py index 6ee2a7c..b046656 100644 --- a/tools/portal_server.py +++ b/tools/portal_server.py @@ -114,8 +114,55 @@ def _commons() -> dict: "cite": r["cite"]} for r in recs]} +def _endpoint() -> str: + """Which surface this portal is — the always-on cloud 'twin' or the 'box' (direct/LAN). Set + SOURCEOS_ENDPOINT=twin on the twin; defaults to box.""" + import os + return os.environ.get("SOURCEOS_ENDPOINT", "box") + + +def _inference() -> dict: + """Sovereign-inference posture: our own models, and where a sensitive prompt would route (never a + cloud LLM). Sovereign endpoints = live trusted GPU backends.""" + inf = _sib("inference") + reg = _registry() + avail = reg.availability() + sovereign_up = [b for b in ("hpc-slurm", "k8s") if avail.get(b, 0) > 0] + models = [inf.model_sphere(name=n, version=v, weights_digest="sha256:" + "ab" * 32, + params_b=p, engine="vllm") + for (n, v, p) in [("llama-3-8b", "q4", 8), ("mixtral-8x7b", "q4", 47), ("nomic-embed", "f16", 0.1)]] + return {"endpoint": _endpoint(), + "posture": "sovereign-first — sensitive inference never leaves for a cloud LLM", + "sovereign_endpoints": sovereign_up, + "models": [{"model": m["model_name"], "params_b": m["params_b"], + "route": inf.route_inference(model=m, sovereign_endpoints=sovereign_up, + prompt_sensitivity="sensitive")["route"]} + for m in models]} + + +_MANIFEST = json.dumps({ + "name": "SourceOS Continuum", "short_name": "Continuum", "start_url": "/", "scope": "/", + "display": "standalone", "background_color": "#0b0d12", "theme_color": "#0b0d12", + "description": "See and reach your infrastructure — twin or box.", + "icons": [{"src": "data:image/svg+xml," + "" + "", + "sizes": "any", "type": "image/svg+xml", "purpose": "any maskable"}]}) + +# cache-first service worker so the console still loads on a flaky mobile link (offline-ish shell). +_SW = ("const C='continuum-v1';" + "self.addEventListener('install',e=>{self.skipWaiting();e.waitUntil(caches.open(C).then(c=>c.add('/')))});" + "self.addEventListener('activate',e=>e.waitUntil(self.clients.claim()));" + "self.addEventListener('fetch',e=>{if(e.request.method!=='GET')return;" + "e.respondWith(fetch(e.request).then(r=>{const cp=r.clone();caches.open(C).then(c=>c.put(e.request,cp));return r})" + ".catch(()=>caches.match(e.request)))});") + + _CONSOLE = """ -SourceOS Continuum — Console +SourceOS Continuum — Console + + + -

SourceOS Continuum — Developer Console

-
Read-only view of the governed surface. Actions run through the MCP surface + fail-closed promotion gate.
+

SourceOS Continuum — Developer Console …

+
Read-only view of the governed surface. Actions run through the MCP surface + fail-closed promotion gate. Installable on mobile; reaches the twin (always-on) or the box (direct/LAN).

Capabilities

loading…

Lifecycle

loading…
@@ -144,6 +191,9 @@ def _commons() -> dict:
Every capability + workload as a citable, content-addressed record (Zenodo-style). reproducible = provenance carries the digests to reproduce it; declared = registered but not yet reproducibility-backed.
loading…
+

Sovereign inference — our own LLMs

+
Models are immutable data spheres served on trusted GPU nodes. A sensitive prompt routes to a sovereign endpoint or blocks — it never leaves for a cloud LLM.
+
loading…

Sealed evidence (latest)

loading…
""" @@ -181,9 +238,14 @@ def route(path: str) -> tuple[int, str, str]: return 200, "text/html; charset=utf-8", _CONSOLE if path == "/healthz": return 200, "text/plain", "ok" + if path == "/manifest.webmanifest": + return 200, "application/manifest+json", _MANIFEST + if path == "/sw.js": + return 200, "application/javascript", _SW api = {"/api/capabilities": _capabilities, "/api/lifecycle": _lifecycle, "/api/evidence": _evidence, "/api/compute": _compute, - "/api/mesh": _mesh, "/api/placements": _placements, "/api/commons": _commons} + "/api/mesh": _mesh, "/api/placements": _placements, "/api/commons": _commons, + "/api/inference": _inference} if path in api: return 200, "application/json", json.dumps(api[path](), indent=2, sort_keys=True) return 404, "text/plain", "not found" diff --git a/tools/test_inference.py b/tools/test_inference.py new file mode 100644 index 0000000..7cf8c62 --- /dev/null +++ b/tools/test_inference.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Tests for sovereign inference. Load-bearing: a model is an immutable data sphere, serving it is a +trusted-GPU workload, and routing NEVER sends sensitive inference to a cloud LLM (fail-closed).""" +import inference as inf + + +def test_model_is_an_immutable_integrity_pinned_data_sphere(): + m = inf.model_sphere(name="llama", version="q4", weights_digest="sha256:" + "ab" * 32, + params_b=8, engine="vllm") + assert m["sphere_id"].startswith("sphere:model/llama@q4+") + assert m["immutable"] is True and m["root_hash"].startswith("sha256:") + assert m["engine"] == "vllm" and m["params_b"] == 8 + + +def test_serving_a_model_is_a_trusted_gpu_workload_referencing_the_sphere(): + m = inf.model_sphere(name="llama", version="q4", weights_digest="sha256:" + "ab" * 32, params_b=8) + wl = inf.inference_service_workload(m, sensitivity="sensitive") + assert wl["needs_gpu"] is True and wl["sensitivity"] == "sensitive" + assert wl["model_sphere"] == m["sphere_id"] + assert wl["needs"]["residency"] == "cluster" + + +def test_routing_prefers_a_sovereign_endpoint(): + m = inf.model_sphere(name="x", version="1", weights_digest="sha256:" + "cd" * 32, params_b=1) + r = inf.route_inference(model=m, sovereign_endpoints=["twin:vllm:8000"]) + assert r["route"] == "sovereign" and r["endpoint"] == "twin:vllm:8000" + + +def test_sensitive_inference_blocks_rather_than_leaking_to_a_cloud_llm(): + m = inf.model_sphere(name="x", version="1", weights_digest="sha256:" + "cd" * 32, params_b=1) + r = inf.route_inference(model=m, sovereign_endpoints=[], prompt_sensitivity="sensitive") + assert r["route"] == "blocked" and r["endpoint"] is None + assert "REFUSING" in r["reason"] + + +def test_residency_fenced_model_forces_sovereign_even_for_normal_prompts(): + m = inf.model_sphere(name="x", version="1", weights_digest="sha256:" + "cd" * 32, params_b=1, + residency="eu") + r = inf.route_inference(model=m, sovereign_endpoints=[], prompt_sensitivity="normal") + assert r["route"] == "blocked" # residency ring-fence overrides "normal" + + +def test_non_sensitive_may_fall_back_to_a_vendor_only_when_allowed(): + m = inf.model_sphere(name="x", version="1", weights_digest="sha256:" + "cd" * 32, params_b=1, + residency="any") + assert inf.route_inference(model=m, sovereign_endpoints=[], prompt_sensitivity="normal", + allow_vendor=True)["route"] == "vendor" + assert inf.route_inference(model=m, sovereign_endpoints=[], prompt_sensitivity="normal", + allow_vendor=False)["route"] == "blocked" + + +def test_inference_service_places_on_a_trusted_gpu_backend(): + import compute_plane as cp + m = inf.model_sphere(name="x", version="1", weights_digest="sha256:" + "cd" * 32, params_b=70, + residency="cluster") + wl = inf.inference_service_workload(m) + d = cp.place(wl, {}, {b: 100 for b in cp.BACKENDS}) + assert d["backend"] in ("hpc-slurm", "k8s") and d["backend_trust"] == "trusted" # never volunteer/vendor + + +if __name__ == "__main__": + import sys + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + for fn in fns: + fn() + print(f"ok: {len(fns)} inference tests passed") + sys.exit(0) diff --git a/tools/test_portal_server.py b/tools/test_portal_server.py index a01250b..37f3787 100644 --- a/tools/test_portal_server.py +++ b/tools/test_portal_server.py @@ -47,6 +47,29 @@ def test_unknown_path_is_404(): assert status == 404 +def test_pwa_manifest_and_service_worker_are_served(): + st, ct, body = ps.route("/manifest.webmanifest") + assert st == 200 and "manifest" in ct and "Continuum" in body and "standalone" in body + st2, ct2, _ = ps.route("/sw.js") + assert st2 == 200 and "javascript" in ct2 + + +def test_console_is_installable_and_shows_sovereign_inference(): + html = ps.route("/")[2] + assert "rel=manifest" in html and "Sovereign inference" in html and "epbadge" in html + + +def test_inference_api_is_sovereign_first_and_fail_closed_without_endpoints(): + with tempfile.TemporaryDirectory() as td: + old, ps._HEARTBEATS = ps._HEARTBEATS, pathlib.Path(td) # no live GPU backend -> no sovereign endpoint + try: + d = json.loads(ps.route("/api/inference")[2]) + assert "endpoint" in d and d["sovereign_endpoints"] == [] + assert all(m["route"] == "blocked" for m in d["models"]) # never a cloud LLM + finally: + ps._HEARTBEATS = old + + def test_devspace_capability_is_surfaced(): caps = json.loads(ps.route("/api/capabilities")[2])["capabilities"] ids = {c.get("capability_id") for c in caps} diff --git a/tools/validate.py b/tools/validate.py index 233a1ce..0f0f9f3 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -25,6 +25,7 @@ "capd/self-healing-loop.mesh.capd.json", "capd/volunteer-mesh-verification.mesh.capd.json", "capd/data-spheres.mesh.capd.json", + "capd/sovereign-inference.mesh.capd.json", "tools/promotion_gate.py", "tools/portal_server.py", "tools/compute_plane.py", @@ -42,6 +43,7 @@ "tools/devmode.py", "tools/data_sphere.py", "tools/availability.py", + "tools/inference.py", ] CAPD_KEYS = ("capability_id", "kind", "status", "links", "composes_with", "policy") # Every CapD in capd/ must carry the core keys and parse — not just the flagship control-plane one. @@ -53,6 +55,7 @@ "capd/self-healing-loop.mesh.capd.json": "caps.compute.self-healing-loop", "capd/volunteer-mesh-verification.mesh.capd.json": "caps.compute.volunteer-mesh-verification", "capd/data-spheres.mesh.capd.json": "caps.data.spheres", + "capd/sovereign-inference.mesh.capd.json": "caps.inference.sovereign", } errors: list[str] = []