From 42f26cfd0504f46c779a2013cf9eb296f1cb7d35 Mon Sep 17 00:00:00 2001 From: Ning Zhou Date: Tue, 28 Jul 2026 23:16:56 +0800 Subject: [PATCH] feat(platform): rehearse Gravitino provider identity --- .github/workflows/ci.yml | 5 + ...adata-fabric-gravitino-identity.local.yaml | 64 + .../metadata_fabric_gravitino_identity.py | 1414 +++++++++++++++++ data_agent/platform_truth.py | 20 + ...test_metadata_fabric_gravitino_identity.py | 392 +++++ data_agent/test_platform_truth.py | 5 + ...avitino-basic-bounded-provider-identity.md | 67 + ...-fabric-gravitino-identity-2026-07-28.json | 152 ++ docs/roadmap-ar0-platform-truth-2026-07-24.md | 5 +- docs/roadmap.md | 2 +- docs/system-of-record-matrix-2026-07-24.md | 10 +- .../kustomization.yaml | 11 + .../namespace.yaml | 23 + .../runtime.yaml | 259 +++ scripts/metadata-fabric-gravitino-identity.sh | 22 + 15 files changed, 2443 insertions(+), 8 deletions(-) create mode 100644 config/metadata-fabric-gravitino-identity.local.yaml create mode 100644 data_agent/metadata_fabric_gravitino_identity.py create mode 100644 data_agent/test_metadata_fabric_gravitino_identity.py create mode 100644 docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md create mode 100644 docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json create mode 100644 k8s/metadata-fabric-gravitino-identity/kustomization.yaml create mode 100644 k8s/metadata-fabric-gravitino-identity/namespace.yaml create mode 100644 k8s/metadata-fabric-gravitino-identity/runtime.yaml create mode 100755 scripts/metadata-fabric-gravitino-identity.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf55ab36..d87e1dfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ on: - feat/ar1-metadata-fabric-binding-ledger - feat/ar1-metadata-fabric-openlineage-delivery - feat/ar1-metadata-fabric-provider-identity + - feat/ar1-metadata-fabric-gravitino-identity env: PYTHON_VERSION: "3.13" @@ -133,6 +134,9 @@ jobs: - name: Validate metadata fabric provider identity evidence run: python -m data_agent.metadata_fabric_provider_identity validate + - name: Validate metadata fabric Gravitino identity evidence + run: python -m data_agent.metadata_fabric_gravitino_identity validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -189,6 +193,7 @@ jobs: data_agent/test_metadata_fabric_binding_contract.py \ data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_provider_identity.py \ + data_agent/test_metadata_fabric_gravitino_identity.py \ data_agent/test_metadata_fabric_otel_failure_rehearsal.py \ data_agent/test_metadata_fabric_otel_metrics.py \ data_agent/test_metadata_fabric_provider_metrics.py \ diff --git a/config/metadata-fabric-gravitino-identity.local.yaml b/config/metadata-fabric-gravitino-identity.local.yaml new file mode 100644 index 00000000..d2d5a6d5 --- /dev/null +++ b/config/metadata-fabric-gravitino-identity.local.yaml @@ -0,0 +1,64 @@ +schema: gda.metadata_fabric_gravitino_identity_profile.v1 +environment: local_docker_desktop + +cluster: + context: docker-desktop + source_namespace: gda-metadata-sandbox + rehearsal_namespace: gda-metadata-identity + source_schema_configmap: metadata-gravitino-schema-1-3-0 + +runtime: + manifest: k8s/metadata-fabric-gravitino-identity + gravitino_version: 1.3.0 + gravitino_image: gda/gravitino:1.3.0-local-arm64 + postgresql_version: 16.10-bookworm + postgresql_image: postgres:16.10-bookworm + service: gravitino-identity + service_port: 8090 + authenticator: basic + idp_extension: org.apache.gravitino.idp.web.rest.feature + access_control_enabled: true + service_account: gravitino-identity + service_account_automount_disabled: true + transport: local_loopback_http + +identity: + service_admin: gda-identity-admin + user: gda-metadata-projection + role: gda-table-projection + material_delivery: runtime_generated_ephemeral_kubernetes_object + login_rotation: administrator_reset + revocation: idp_user_delete + +scope: + metalake: gda_identity + catalog: lakehouse + schema: published + table: gda_identity_probe + denied_catalog: unauthorized_catalog + role_securable_objects: + - full_name: lakehouse + type: CATALOG + privileges: + - name: USE_CATALOG + condition: ALLOW + - full_name: lakehouse.published + type: SCHEMA + privileges: + - name: CREATE_TABLE + condition: ALLOW + - name: USE_SCHEMA + condition: ALLOW + +claims: + local_gravitino_basic_identity_verified: false + local_gravitino_minimum_privilege_verified: false + local_gravitino_login_rotation_verified: false + local_gravitino_revocation_verified: false + gravitino_authentication_verified: false + provider_minimum_privilege_verified: false + protected_workload_identity_verified: false + oidc_verified: false + tls_verified: false + production_identity_verified: false + production_ready: false diff --git a/data_agent/metadata_fabric_gravitino_identity.py b/data_agent/metadata_fabric_gravitino_identity.py new file mode 100644 index 00000000..6c9de899 --- /dev/null +++ b/data_agent/metadata_fabric_gravitino_identity.py @@ -0,0 +1,1414 @@ +"""Rehearse authenticated, bounded Gravitino access in an isolated namespace. + +The rehearsal starts a temporary Gravitino 1.3.0 server backed by PostgreSQL, +enables the built-in Basic IdP and deny-by-default authorization, and grants one +named user only the privileges needed to create a table in one schema. It then +proves an allowed table mutation, a denied catalog mutation, login rotation, +revocation, and complete namespace cleanup. This remains local POC evidence, +not protected OIDC or production identity. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import secrets +import subprocess +import sys +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Annotated, Any, Literal +from urllib.parse import quote +from uuid import UUID + +import httpx +import yaml +from pydantic import BaseModel, ConfigDict, Field, SecretStr, StringConstraints + +from . import metadata_fabric_ingestion_replay as ingestion_replay +from . import metadata_fabric_provider_metrics as provider_metrics +from . import metadata_fabric_recovery_rehearsal as recovery + + +PROFILE_SCHEMA = "gda.metadata_fabric_gravitino_identity_profile.v1" +CONTRACT_SCHEMA = "gda.metadata_fabric_gravitino_identity_contract.v1" +OBSERVATION_SCHEMA = "gda.metadata_fabric_gravitino_identity_observation.v1" +EVIDENCE_SCHEMA = "gda.metadata_fabric_gravitino_identity_evidence.v1" +VALIDATION_SCHEMA = "gda.metadata_fabric_gravitino_identity_validation.v1" + +CONTEXT = "docker-desktop" +SOURCE_NAMESPACE = "gda-metadata-sandbox" +REHEARSAL_NAMESPACE = "gda-metadata-identity" +GRAVITINO_SCHEMA_SHA256 = ( + "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df" +) + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = ( + REPO_ROOT / "config/metadata-fabric-gravitino-identity.local.yaml" +) +DEFAULT_EVIDENCE_PATH = ( + REPO_ROOT / "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json" +) +DEFAULT_WRAPPER_PATH = REPO_ROOT / "scripts/metadata-fabric-gravitino-identity.sh" +MANIFEST_DIR = REPO_ROOT / "k8s/metadata-fabric-gravitino-identity" + +NonEmptyText = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=1024), +] + + +class MetadataFabricGravitinoIdentityError(RuntimeError): + """The Gravitino identity contract or live rehearsal failed closed.""" + + +class ProviderRequestError(MetadataFabricGravitinoIdentityError): + """A Gravitino request failed without exposing response or login material.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ClusterProfile(_FrozenModel): + context: Literal["docker-desktop"] + source_namespace: Literal["gda-metadata-sandbox"] + rehearsal_namespace: Literal["gda-metadata-identity"] + source_schema_configmap: Literal["metadata-gravitino-schema-1-3-0"] + + +class RuntimeProfile(_FrozenModel): + manifest: Literal["k8s/metadata-fabric-gravitino-identity"] + gravitino_version: Literal["1.3.0"] + gravitino_image: Literal["gda/gravitino:1.3.0-local-arm64"] + postgresql_version: Literal["16.10-bookworm"] + postgresql_image: Literal["postgres:16.10-bookworm"] + service: Literal["gravitino-identity"] + service_port: Literal[8090] + authenticator: Literal["basic"] + idp_extension: Literal["org.apache.gravitino.idp.web.rest.feature"] + access_control_enabled: Literal[True] + service_account: Literal["gravitino-identity"] + service_account_automount_disabled: Literal[True] + transport: Literal["local_loopback_http"] + + +class IdentityProfile(_FrozenModel): + service_admin: Literal["gda-identity-admin"] + user: Literal["gda-metadata-projection"] + role: Literal["gda-table-projection"] + material_delivery: Literal["runtime_generated_ephemeral_kubernetes_object"] + login_rotation: Literal["administrator_reset"] + revocation: Literal["idp_user_delete"] + + +class PrivilegeProfile(_FrozenModel): + name: Literal["USE_CATALOG", "USE_SCHEMA", "CREATE_TABLE"] + condition: Literal["ALLOW"] + + +class SecurableObjectProfile(_FrozenModel): + full_name: Literal["lakehouse", "lakehouse.published"] + type: Literal["CATALOG", "SCHEMA"] + privileges: tuple[PrivilegeProfile, ...] + + +class ScopeProfile(_FrozenModel): + metalake: Literal["gda_identity"] + catalog: Literal["lakehouse"] + schema_name: Literal["published"] = Field(alias="schema") + table: Literal["gda_identity_probe"] + denied_catalog: Literal["unauthorized_catalog"] + role_securable_objects: tuple[SecurableObjectProfile, ...] + + +class ClaimProfile(_FrozenModel): + local_gravitino_basic_identity_verified: Literal[False] + local_gravitino_minimum_privilege_verified: Literal[False] + local_gravitino_login_rotation_verified: Literal[False] + local_gravitino_revocation_verified: Literal[False] + gravitino_authentication_verified: Literal[False] + provider_minimum_privilege_verified: Literal[False] + protected_workload_identity_verified: Literal[False] + oidc_verified: Literal[False] + tls_verified: Literal[False] + production_identity_verified: Literal[False] + production_ready: Literal[False] + + +class GravitinoIdentityProfile(_FrozenModel): + schema_name: Literal["gda.metadata_fabric_gravitino_identity_profile.v1"] = ( + Field(alias="schema") + ) + environment: Literal["local_docker_desktop"] + cluster: ClusterProfile + runtime: RuntimeProfile + identity: IdentityProfile + scope: ScopeProfile + claims: ClaimProfile + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _valid_uuid(value: Any) -> bool: + try: + UUID(str(value)) + except (TypeError, ValueError): + return False + return True + + +def _valid_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _expected_securable_objects() -> list[dict[str, Any]]: + return [ + { + "fullName": "lakehouse", + "type": "CATALOG", + "privileges": [{"name": "USE_CATALOG", "condition": "ALLOW"}], + }, + { + "fullName": "lakehouse.published", + "type": "SCHEMA", + "privileges": [ + {"name": "CREATE_TABLE", "condition": "ALLOW"}, + {"name": "USE_SCHEMA", "condition": "ALLOW"}, + ], + }, + ] + + +def _profile_securable_objects( + profile: GravitinoIdentityProfile, +) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for item in profile.scope.role_securable_objects: + result.append( + { + "fullName": item.full_name, + "type": item.type, + "privileges": sorted( + [entry.model_dump(mode="json") for entry in item.privileges], + key=lambda entry: entry["name"], + ), + } + ) + return sorted(result, key=lambda item: item["fullName"]) + + +def load_profile(path: Path = DEFAULT_PROFILE_PATH) -> GravitinoIdentityProfile: + try: + value = yaml.safe_load(path.resolve().read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino identity profile must be an object") + ingestion_replay._reject_sensitive_fields(value) + profile = GravitinoIdentityProfile.model_validate(value) + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity profile is invalid" + ) from exc + if _profile_securable_objects(profile) != _expected_securable_objects(): + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity role exceeds the bounded table-create scope" + ) + return profile + + +def _manifest_documents() -> list[dict[str, Any]]: + documents: list[dict[str, Any]] = [] + for path in sorted(MANIFEST_DIR.glob("*.yaml")): + if path.name == "kustomization.yaml": + continue + for value in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if isinstance(value, dict): + documents.append(value) + return documents + + +def _validate_manifest() -> list[str]: + errors: list[str] = [] + try: + documents = _manifest_documents() + except (OSError, yaml.YAMLError) as exc: + return [f"Gravitino identity manifest is invalid: {type(exc).__name__}"] + if any(document.get("kind") == "Secret" for document in documents): + errors.append("Gravitino identity manifest may not commit Secret values") + kinds = {str(document.get("kind")) for document in documents} + if not {"Namespace", "ServiceAccount", "ConfigMap", "Service", "StatefulSet"}.issubset( + kinds + ): + errors.append("Gravitino identity manifest is incomplete") + rendered = json.dumps(documents, ensure_ascii=True, sort_keys=True) + for marker in ( + "gravitino.authenticators = basic", + "gravitino.authorization.enable = true", + "org.apache.gravitino.idp.web.rest.feature", + "GRAVITINO_INITIAL_ADMIN_PASSWORD", + "automountServiceAccountToken", + "ClusterIP", + "emptyDir", + ): + if marker not in rendered: + errors.append(f"Gravitino identity manifest is missing marker: {marker}") + if "gravitino.authenticators = simple" in rendered: + errors.append("Gravitino identity manifest may not enable the simple authenticator") + return errors + + +def build_contract_report( + profile_path: Path = DEFAULT_PROFILE_PATH, + wrapper_path: Path = DEFAULT_WRAPPER_PATH, +) -> dict[str, Any]: + errors: list[str] = [] + profile: GravitinoIdentityProfile | None = None + try: + profile = load_profile(profile_path) + except MetadataFabricGravitinoIdentityError as exc: + errors.append(str(exc)) + errors.extend(_validate_manifest()) + + try: + wrapper = wrapper_path.resolve().read_text(encoding="utf-8") + for marker in ("set -euo pipefail", "metadata_fabric_gravitino_identity"): + if marker not in wrapper: + errors.append(f"Gravitino identity wrapper is missing marker: {marker}") + except OSError as exc: + errors.append(f"Gravitino identity wrapper is invalid: {type(exc).__name__}") + + files: dict[str, dict[str, str]] = {} + paths = [Path(__file__).resolve(), profile_path.resolve(), wrapper_path.resolve()] + paths.extend(sorted(MANIFEST_DIR.glob("*.yaml"))) + for path in paths: + if not path.is_file(): + continue + try: + relative = path.relative_to(REPO_ROOT).as_posix() + except ValueError: + relative = path.name + files[relative] = {"path": relative, "sha256": recovery._file_sha256(path)} + + stable = { + "schema": CONTRACT_SCHEMA, + "context": CONTEXT, + "source_namespace": SOURCE_NAMESPACE, + "rehearsal_namespace": REHEARSAL_NAMESPACE, + "gravitino_version": "1.3.0", + "authentication": { + "authenticator": profile.runtime.authenticator if profile else None, + "access_control_enabled": ( + profile.runtime.access_control_enabled if profile else None + ), + "idp_extension": profile.runtime.idp_extension if profile else None, + "simple_authenticator_trusted": False, + "built_in_idp_scope": "local_poc_only", + }, + "role_securable_objects": ( + _profile_securable_objects(profile) if profile else None + ), + "local_static_contract_verified": not errors, + "local_gravitino_basic_identity_verified": False, + "local_gravitino_minimum_privilege_verified": False, + "gravitino_authentication_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "production_identity_verified": False, + "production_ready": False, + "files": files, + "errors": errors, + } + return {**stable, "contract_fingerprint": recovery._canonical_sha256(stable)} + + +class _Kubectl: + def __init__(self, context: str) -> None: + self.context = context + + def run( + self, + args: list[str], + *, + input_text: str | None = None, + expected: frozenset[int] = frozenset({0}), + timeout: int = 300, + label: str, + ) -> subprocess.CompletedProcess[str]: + try: + completed = subprocess.run( + ["kubectl", "--context", self.context, *args], + input=input_text, + capture_output=True, + check=False, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise MetadataFabricGravitinoIdentityError( + f"kubectl failed during {label}" + ) from exc + if completed.returncode not in expected: + raise MetadataFabricGravitinoIdentityError( + f"kubectl returned {completed.returncode} during {label}" + ) + return completed + + def get_json( + self, + args: list[str], + *, + allow_not_found: bool = False, + label: str, + ) -> dict[str, Any] | None: + expected = frozenset({0, 1}) if allow_not_found else frozenset({0}) + completed = self.run( + [*args, "-o", "json"], expected=expected, timeout=60, label=label + ) + if completed.returncode != 0: + return None + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise MetadataFabricGravitinoIdentityError( + f"kubectl returned invalid JSON during {label}" + ) from exc + if not isinstance(value, dict): + raise MetadataFabricGravitinoIdentityError( + f"kubectl returned a non-object during {label}" + ) + return value + + +class IsolatedGravitinoRuntime: + """Own and remove the temporary namespace used by the identity rehearsal.""" + + def __init__(self, profile: GravitinoIdentityProfile) -> None: + self.profile = profile + self.kubectl = _Kubectl(profile.cluster.context) + self.owned_namespace = False + self.schema_sha256: str | None = None + + def _runtime_inputs( + self, + admin_material: SecretStr, + database_material: SecretStr, + ) -> str: + source = self.kubectl.get_json( + [ + "-n", + self.profile.cluster.source_namespace, + "get", + "configmap", + self.profile.cluster.source_schema_configmap, + ], + label="source schema lookup", + ) + assert source is not None + schema_sql = _mapping(source.get("data")).get("001-schema.sql") + if not isinstance(schema_sql, str): + raise MetadataFabricGravitinoIdentityError( + "verified Gravitino PostgreSQL schema is unavailable" + ) + self.schema_sha256 = _sha256_text(schema_sql) + if self.schema_sha256 != GRAVITINO_SCHEMA_SHA256: + raise MetadataFabricGravitinoIdentityError( + "Gravitino PostgreSQL schema checksum drift" + ) + resources = { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "gravitino-identity-runtime", + "namespace": self.profile.cluster.rehearsal_namespace, + }, + "type": "Opaque", + "stringData": { + "admin-password": admin_material.get_secret_value(), + "database-password": database_material.get_secret_value(), + }, + }, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "gravitino-identity-schema", + "namespace": self.profile.cluster.rehearsal_namespace, + }, + "data": {"001-schema.sql": schema_sql}, + }, + ], + } + return json.dumps(resources, ensure_ascii=True, separators=(",", ":")) + + def start( + self, + *, + admin_material: SecretStr, + database_material: SecretStr, + ) -> dict[str, Any]: + existing = self.kubectl.get_json( + ["get", "namespace", self.profile.cluster.rehearsal_namespace], + allow_not_found=True, + label="rehearsal namespace preflight", + ) + if existing is not None: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity rehearsal namespace already exists" + ) + self.kubectl.run( + ["apply", "-f", str(MANIFEST_DIR / "namespace.yaml")], + label="rehearsal namespace apply", + ) + self.owned_namespace = True + runtime_inputs = self._runtime_inputs(admin_material, database_material) + self.kubectl.run( + ["apply", "-f", "-"], + input_text=runtime_inputs, + label="ephemeral runtime input apply", + ) + self.kubectl.run( + ["apply", "-k", str(MANIFEST_DIR)], + label="isolated Gravitino runtime apply", + ) + namespace = self.profile.cluster.rehearsal_namespace + for workload in ( + "statefulset/gravitino-identity-postgresql", + "statefulset/gravitino-identity", + ): + self.kubectl.run( + [ + "-n", + namespace, + "rollout", + "status", + workload, + "--timeout=10m", + ], + timeout=660, + label=f"{workload} rollout", + ) + return self.observe() + + def observe(self) -> dict[str, Any]: + namespace_name = self.profile.cluster.rehearsal_namespace + namespace = self.kubectl.get_json( + ["get", "namespace", namespace_name], label="namespace observation" + ) + service = self.kubectl.get_json( + ["-n", namespace_name, "get", "service", "gravitino-identity"], + label="service observation", + ) + workload = self.kubectl.get_json( + [ + "-n", + namespace_name, + "get", + "statefulset", + "gravitino-identity", + ], + label="Gravitino workload observation", + ) + database = self.kubectl.get_json( + [ + "-n", + namespace_name, + "get", + "statefulset", + "gravitino-identity-postgresql", + ], + label="PostgreSQL workload observation", + ) + assert namespace is not None and service is not None + assert workload is not None and database is not None + workload_spec = _mapping(_mapping(workload.get("spec")).get("template")) + workload_pod = _mapping(workload_spec.get("spec")) + containers = workload_pod.get("containers") + container = _mapping(containers[0]) if isinstance(containers, list) else {} + database_spec = _mapping(_mapping(database.get("spec")).get("template")) + database_pod = _mapping(database_spec.get("spec")) + database_containers = database_pod.get("containers") + database_container = ( + _mapping(database_containers[0]) + if isinstance(database_containers, list) + else {} + ) + data_volume = next( + ( + _mapping(item) + for item in database_pod.get("volumes", []) + if _mapping(item).get("name") == "data" + ), + {}, + ) + return { + "context": self.profile.cluster.context, + "namespace": { + "name": _mapping(namespace.get("metadata")).get("name"), + "uid": _mapping(namespace.get("metadata")).get("uid"), + }, + "service": { + "name": _mapping(service.get("metadata")).get("name"), + "uid": _mapping(service.get("metadata")).get("uid"), + "type": _mapping(service.get("spec")).get("type"), + }, + "gravitino": { + "kind": workload.get("kind"), + "name": _mapping(workload.get("metadata")).get("name"), + "uid": _mapping(workload.get("metadata")).get("uid"), + "image": container.get("image"), + "service_account": workload_pod.get("serviceAccountName"), + "service_account_automount_disabled": ( + workload_pod.get("automountServiceAccountToken") is False + ), + "ready_replicas": _mapping(workload.get("status")).get( + "readyReplicas", 0 + ), + }, + "postgresql": { + "kind": database.get("kind"), + "name": _mapping(database.get("metadata")).get("name"), + "uid": _mapping(database.get("metadata")).get("uid"), + "image": database_container.get("image"), + "service_account": database_pod.get("serviceAccountName"), + "service_account_automount_disabled": ( + database_pod.get("automountServiceAccountToken") is False + ), + "ready_replicas": _mapping(database.get("status")).get( + "readyReplicas", 0 + ), + "ephemeral_data_volume": "emptyDir" in data_volume, + }, + "source_schema_sha256": self.schema_sha256, + } + + def cleanup(self) -> dict[str, Any]: + deleted = False + if self.owned_namespace: + try: + self.kubectl.run( + [ + "delete", + "namespace", + self.profile.cluster.rehearsal_namespace, + "--wait=true", + "--timeout=5m", + ], + timeout=330, + label="rehearsal namespace cleanup", + ) + deleted = True + finally: + self.owned_namespace = False + absent = ( + self.kubectl.get_json( + ["get", "namespace", self.profile.cluster.rehearsal_namespace], + allow_not_found=True, + label="rehearsal namespace cleanup verification", + ) + is None + ) + return { + "namespace_delete_completed": deleted, + "namespace_absent": absent, + "provider_objects_retained": False, + } + + +def _json_object(response: httpx.Response, label: str) -> dict[str, Any]: + try: + value = response.json() + except ValueError as exc: + raise ProviderRequestError(f"{label} returned invalid JSON") from exc + if not isinstance(value, dict): + raise ProviderRequestError(f"{label} response is not an object") + return value + + +class _BasicApi: + def __init__( + self, + *, + base_url: str, + username: str, + material: SecretStr, + transport: httpx.BaseTransport | None = None, + ) -> None: + self._client = httpx.Client( + base_url=base_url.rstrip("/") + "/", + auth=httpx.BasicAuth(username, material.get_secret_value()), + headers={ + "Accept": "application/vnd.gravitino.v1+json", + "Content-Type": "application/json", + }, + timeout=30.0, + transport=transport, + ) + + def close(self) -> None: + self._client.close() + + def request( + self, + method: str, + path: str, + *, + json_body: Mapping[str, Any] | None = None, + params: Mapping[str, str] | None = None, + expected: frozenset[int] = frozenset({200}), + label: str, + ) -> tuple[int, dict[str, Any] | None]: + try: + response = self._client.request( + method, + path, + json=dict(json_body) if json_body is not None else None, + params=params, + ) + except httpx.HTTPError as exc: + raise ProviderRequestError(f"{label} request failed") from exc + if response.status_code not in expected: + raise ProviderRequestError( + f"{label} returned unexpected status {response.status_code}" + ) + if response.status_code in {401, 403, 404} or not response.content: + return response.status_code, None + return response.status_code, _json_object(response, label) + + +def _response_entity( + payload: Mapping[str, Any] | None, key: str, label: str +) -> dict[str, Any]: + entity = _mapping(_mapping(payload).get(key)) + if not entity: + raise ProviderRequestError(f"{label} omitted {key}") + return dict(entity) + + +def _normalize_securable_objects(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + normalized: list[dict[str, Any]] = [] + for item in value: + source = _mapping(item) + privileges = source.get("privileges") + normalized_privileges = [] + if isinstance(privileges, list): + for privilege in privileges: + entry = _mapping(privilege) + normalized_privileges.append( + { + "name": str(entry.get("name", "")).upper(), + "condition": str(entry.get("condition", "")).upper(), + } + ) + normalized.append( + { + "fullName": source.get("fullName"), + "type": str(source.get("type", "")).upper(), + "privileges": sorted( + normalized_privileges, key=lambda entry: str(entry["name"]) + ), + } + ) + return sorted(normalized, key=lambda entry: str(entry["fullName"])) + + +class GravitinoIdentityRehearsal: + """Exercise one Basic IdP user with a schema-bounded table-create role.""" + + def __init__( + self, + *, + base_url: str, + admin_name: str, + admin_material: SecretStr, + transport: httpx.BaseTransport | None = None, + ) -> None: + self.base_url = base_url + self.transport = transport + self.admin = _BasicApi( + base_url=base_url, + username=admin_name, + material=admin_material, + transport=transport, + ) + self.clients: list[_BasicApi] = [] + + def close(self) -> None: + for client in self.clients: + client.close() + self.clients.clear() + self.admin.close() + + def _user_api(self, name: str, material: SecretStr) -> _BasicApi: + client = _BasicApi( + base_url=self.base_url, + username=name, + material=material, + transport=self.transport, + ) + self.clients.append(client) + return client + + @staticmethod + def _catalog_path(profile: GravitinoIdentityProfile, catalog: str) -> str: + return f"metalakes/{quote(profile.scope.metalake)}/catalogs/{quote(catalog)}" + + @staticmethod + def _schema_path(profile: GravitinoIdentityProfile) -> str: + return ( + f"metalakes/{quote(profile.scope.metalake)}/catalogs/" + f"{quote(profile.scope.catalog)}/schemas/{quote(profile.scope.schema_name)}" + ) + + @classmethod + def _table_path(cls, profile: GravitinoIdentityProfile) -> str: + return f"{cls._schema_path(profile)}/tables/{quote(profile.scope.table)}" + + def execute( + self, + profile: GravitinoIdentityProfile, + *, + initial_material: SecretStr, + rotated_material: SecretStr, + ) -> dict[str, Any]: + admin_status, version_payload = self.admin.request( + "GET", "version", label="Gravitino service-admin authentication" + ) + version = _mapping(_mapping(version_payload).get("version")).get("version") + if version is None: + version = _mapping(version_payload).get("version") + + unregistered = self._user_api(profile.identity.user, initial_material) + unregistered_status, _payload = unregistered.request( + "GET", + "version", + expected=frozenset({200, 401}), + label="Gravitino unregistered principal probe", + ) + + _status, metalake_payload = self.admin.request( + "POST", + "metalakes", + json_body={ + "name": profile.scope.metalake, + "comment": "Ephemeral GDA authenticated identity rehearsal", + "properties": {"gda.environment": "local_identity_rehearsal"}, + }, + label="Gravitino metalake create", + ) + metalake = _response_entity(metalake_payload, "metalake", "metalake create") + _status, catalog_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body={ + "name": profile.scope.catalog, + "type": "RELATIONAL", + "provider": "lakehouse-iceberg", + "comment": "Ephemeral authenticated table projection catalog", + "properties": { + "catalog-backend": "memory", + "uri": "file:///tmp/gda-identity", + "warehouse": "file:///tmp/gda-identity", + }, + }, + label="Gravitino catalog create", + ) + catalog = _response_entity(catalog_payload, "catalog", "catalog create") + _status, schema_payload = self.admin.request( + "POST", + f"{self._catalog_path(profile, profile.scope.catalog)}/schemas", + json_body={ + "name": profile.scope.schema_name, + "comment": "Ephemeral bounded table projection schema", + "properties": {}, + }, + label="Gravitino schema create", + ) + schema = _response_entity(schema_payload, "schema", "schema create") + + _status, idp_user_payload = self.admin.request( + "POST", + "idp/users", + json_body={ + "user": profile.identity.user, + "password": initial_material.get_secret_value(), + }, + label="Gravitino built-in IdP user create", + ) + _status, user_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/users", + json_body={"name": profile.identity.user}, + label="Gravitino metalake user register", + ) + user = _response_entity(user_payload, "user", "metalake user register") + + _status, role_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/roles", + json_body={ + "name": profile.identity.role, + "properties": {"gda.scope": "bounded_table_projection"}, + "securableObjects": _expected_securable_objects(), + }, + label="Gravitino bounded role create", + ) + role = _response_entity(role_payload, "role", "role create") + _status, granted_payload = self.admin.request( + "PUT", + ( + f"metalakes/{quote(profile.scope.metalake)}/permissions/users/" + f"{quote(profile.identity.user)}/grant" + ), + json_body={"roleNames": [profile.identity.role]}, + label="Gravitino role grant", + ) + granted_user = _response_entity(granted_payload, "user", "role grant") + _status, role_read_payload = self.admin.request( + "GET", + ( + f"metalakes/{quote(profile.scope.metalake)}/roles/" + f"{quote(profile.identity.role)}" + ), + label="Gravitino role readback", + ) + role_read = _response_entity(role_read_payload, "role", "role readback") + + initial_api = self._user_api(profile.identity.user, initial_material) + initial_status, _initial_payload = initial_api.request( + "GET", "version", label="Gravitino bounded user authentication" + ) + create_status, table_payload = initial_api.request( + "POST", + f"{self._schema_path(profile)}/tables", + json_body={ + "name": profile.scope.table, + "comment": "Ephemeral authenticated table-create probe", + "columns": [ + { + "name": "probe_id", + "type": "string", + "nullable": False, + "comment": "Identity probe identifier", + } + ], + "properties": {"gda.identity_probe": "true"}, + }, + label="Gravitino allowed table create probe", + ) + table = _response_entity(table_payload, "table", "table create") + read_status, table_read_payload = initial_api.request( + "GET", self._table_path(profile), label="Gravitino table readback" + ) + table_read = _response_entity(table_read_payload, "table", "table readback") + + denied_status, _denied_payload = initial_api.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body={ + "name": profile.scope.denied_catalog, + "type": "RELATIONAL", + "provider": "lakehouse-iceberg", + "comment": "Must not be created by the bounded user", + "properties": { + "catalog-backend": "memory", + "uri": "file:///tmp/gda-identity-denied", + "warehouse": "file:///tmp/gda-identity-denied", + }, + }, + expected=frozenset({200, 403}), + label="Gravitino denied catalog create probe", + ) + + self.admin.request( + "PUT", + f"idp/users/{quote(profile.identity.user)}", + json_body={"password": rotated_material.get_secret_value()}, + label="Gravitino built-in IdP login rotation", + ) + old_after_rotation, _payload = initial_api.request( + "GET", + "version", + expected=frozenset({200, 401}), + label="Gravitino old login after rotation", + ) + rotated_api = self._user_api(profile.identity.user, rotated_material) + rotated_status, rotated_table_payload = rotated_api.request( + "GET", + self._table_path(profile), + expected=frozenset({200, 401, 403}), + label="Gravitino rotated login", + ) + rotated_table = ( + _response_entity(rotated_table_payload, "table", "rotated table readback") + if rotated_status == 200 + else {} + ) + self.admin.request( + "DELETE", + f"idp/users/{quote(profile.identity.user)}", + label="Gravitino built-in IdP user revocation", + ) + after_revocation, _payload = rotated_api.request( + "GET", + "version", + expected=frozenset({200, 401}), + label="Gravitino revoked login", + ) + idp_lookup_status, _payload = self.admin.request( + "GET", + f"idp/users/{quote(profile.identity.user)}", + expected=frozenset({200, 404}), + label="Gravitino built-in IdP user absence", + ) + + return { + "version": version, + "configuration": { + "authenticator": profile.runtime.authenticator, + "access_control_enabled": profile.runtime.access_control_enabled, + "idp_extension": profile.runtime.idp_extension, + "built_in_idp_scope": "local_poc_only", + "simple_authenticator_trusted": False, + }, + "authentication": { + "service_admin_status": admin_status, + "unregistered_principal_status": unregistered_status, + "bounded_user_status": initial_status, + "material_recorded": False, + }, + "principal": { + "name": user.get("name"), + "granted_name": granted_user.get("name"), + "roles": sorted(granted_user.get("roles", [])), + "is_service_admin": False, + }, + "role": { + "name": role.get("name"), + "readback_name": role_read.get("name"), + "securable_objects": _normalize_securable_objects( + role_read.get("securableObjects") + ), + }, + "bootstrap": { + "metalake": metalake.get("name"), + "catalog": catalog.get("name"), + "schema": schema.get("name"), + "service_admin_used": True, + "minimum_privilege": False, + }, + "allowed_probe": { + "operation": "CREATE_TABLE", + "resource": "SCHEMA", + "create_status": create_status, + "read_status": read_status, + "table": table.get("name"), + "readback_table": table_read.get("name"), + "rotated_readback_table": rotated_table.get("name"), + "full_name": ( + f"{profile.scope.catalog}.{profile.scope.schema_name}." + f"{profile.scope.table}" + ), + }, + "denied_probe": { + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "catalog": profile.scope.denied_catalog, + "status": denied_status, + }, + "login_lifecycle": { + "rotation_mode": profile.identity.login_rotation, + "old_after_rotation_status": old_after_rotation, + "rotated_authentication_status": rotated_status, + "revocation_mode": profile.identity.revocation, + "after_revocation_status": after_revocation, + "idp_principal_absent": idp_lookup_status == 404, + "material_recorded": False, + }, + "idp_create_response_present": bool(idp_user_payload), + } + + +def build_evidence(observation: Mapping[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + try: + ingestion_replay._reject_sensitive_fields(observation) + except ValueError: + errors.append("Gravitino identity observation contains sensitive material") + if observation.get("schema") != OBSERVATION_SCHEMA: + errors.append("Gravitino identity observation schema does not match") + + contract = _mapping(observation.get("contract")) + if ( + contract.get("local_static_contract_verified") is not True + or not _valid_sha256(contract.get("contract_fingerprint")) + ): + errors.append("Gravitino identity static contract is not bound") + + runtime = _mapping(observation.get("runtime")) + gravitino_runtime = _mapping(runtime.get("gravitino")) + postgresql_runtime = _mapping(runtime.get("postgresql")) + if ( + runtime.get("context") != CONTEXT + or _mapping(runtime.get("namespace")).get("name") != REHEARSAL_NAMESPACE + or not _valid_uuid(_mapping(runtime.get("namespace")).get("uid")) + or _mapping(runtime.get("service")).get("name") != "gravitino-identity" + or _mapping(runtime.get("service")).get("type") != "ClusterIP" + or not _valid_uuid(_mapping(runtime.get("service")).get("uid")) + or runtime.get("source_schema_sha256") != GRAVITINO_SCHEMA_SHA256 + or gravitino_runtime.get("kind") != "StatefulSet" + or gravitino_runtime.get("name") != "gravitino-identity" + or gravitino_runtime.get("image") != "gda/gravitino:1.3.0-local-arm64" + or gravitino_runtime.get("service_account") != "gravitino-identity" + or gravitino_runtime.get("service_account_automount_disabled") is not True + or gravitino_runtime.get("ready_replicas") != 1 + or postgresql_runtime.get("kind") != "StatefulSet" + or postgresql_runtime.get("name") != "gravitino-identity-postgresql" + or postgresql_runtime.get("image") != "postgres:16.10-bookworm" + or postgresql_runtime.get("service_account") + != "gravitino-identity-postgresql" + or postgresql_runtime.get("service_account_automount_disabled") is not True + or postgresql_runtime.get("ready_replicas") != 1 + or postgresql_runtime.get("ephemeral_data_volume") is not True + ): + errors.append("isolated Gravitino identity runtime does not match") + + provider = _mapping(observation.get("gravitino")) + configuration = _mapping(provider.get("configuration")) + if configuration != { + "authenticator": "basic", + "access_control_enabled": True, + "idp_extension": "org.apache.gravitino.idp.web.rest.feature", + "built_in_idp_scope": "local_poc_only", + "simple_authenticator_trusted": False, + }: + errors.append("Gravitino Basic IdP configuration does not match") + if provider.get("version") != "1.3.0": + errors.append("Gravitino identity provider version does not match") + + authentication = _mapping(provider.get("authentication")) + if authentication != { + "service_admin_status": 200, + "unregistered_principal_status": 401, + "bounded_user_status": 200, + "material_recorded": False, + }: + errors.append("Gravitino Basic authentication was not enforced") + + principal = _mapping(provider.get("principal")) + if ( + principal.get("name") != "gda-metadata-projection" + or principal.get("granted_name") != "gda-metadata-projection" + or principal.get("roles") != ["gda-table-projection"] + or principal.get("is_service_admin") is not False + ): + errors.append("Gravitino bounded principal does not match") + + role = _mapping(provider.get("role")) + if ( + role.get("name") != "gda-table-projection" + or role.get("readback_name") != "gda-table-projection" + or role.get("securable_objects") != _expected_securable_objects() + ): + errors.append("Gravitino role exceeds the bounded table-create scope") + + bootstrap = _mapping(provider.get("bootstrap")) + if bootstrap != { + "metalake": "gda_identity", + "catalog": "lakehouse", + "schema": "published", + "service_admin_used": True, + "minimum_privilege": False, + }: + errors.append("Gravitino service-admin bootstrap boundary is missing") + + allowed = _mapping(provider.get("allowed_probe")) + if ( + allowed.get("operation") != "CREATE_TABLE" + or allowed.get("resource") != "SCHEMA" + or allowed.get("create_status") != 200 + or allowed.get("read_status") != 200 + or allowed.get("table") != "gda_identity_probe" + or allowed.get("readback_table") != "gda_identity_probe" + or allowed.get("rotated_readback_table") != "gda_identity_probe" + or allowed.get("full_name") != "lakehouse.published.gda_identity_probe" + ): + errors.append("Gravitino allowed table-create probe did not pass") + + denied = _mapping(provider.get("denied_probe")) + if denied != { + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "catalog": "unauthorized_catalog", + "status": 403, + }: + errors.append("Gravitino catalog-create denial was not enforced") + + lifecycle = _mapping(provider.get("login_lifecycle")) + if lifecycle != { + "rotation_mode": "administrator_reset", + "old_after_rotation_status": 401, + "rotated_authentication_status": 200, + "revocation_mode": "idp_user_delete", + "after_revocation_status": 401, + "idp_principal_absent": True, + "material_recorded": False, + }: + errors.append("Gravitino login rotation or revocation did not pass") + if provider.get("idp_create_response_present") is not True: + errors.append("Gravitino built-in IdP user creation was not observed") + + runtime_checks = _mapping(observation.get("runtime_checks")) + if runtime_checks != { + "namespace_delete_completed": True, + "namespace_absent": True, + "provider_objects_retained": False, + "all_port_forwards_stopped": True, + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }: + errors.append("Gravitino identity runtime cleanup is incomplete") + + verified = not errors + stable = { + "schema": EVIDENCE_SCHEMA, + "status": "local_gravitino_basic_identity_verified" if verified else "blocked", + "observation": dict(observation), + "errors": errors, + "local_gravitino_basic_identity_verified": verified, + "local_gravitino_minimum_privilege_verified": verified, + "local_gravitino_login_rotation_verified": verified, + "local_gravitino_revocation_verified": verified, + "gravitino_authentication_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "production_identity_verified": False, + "production_ready": False, + } + return {**stable, "evidence_fingerprint": recovery._canonical_sha256(stable)} + + +def verify_evidence_integrity(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if evidence.get("schema") != EVIDENCE_SCHEMA: + errors.append("Gravitino identity evidence schema does not match") + expected = build_evidence(_mapping(evidence.get("observation"))) + if dict(evidence) != expected: + errors.append("Gravitino identity evidence content or fingerprint drifted") + for claim in ( + "gravitino_authentication_verified", + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "production_identity_verified", + "production_ready", + ): + if evidence.get(claim) is not False: + errors.append(f"Gravitino identity evidence may not claim {claim}") + if evidence.get("local_gravitino_basic_identity_verified") is not True: + errors.append("local Gravitino Basic identity is not verified") + return errors + + +def run_live_rehearsal( + profile_path: Path = DEFAULT_PROFILE_PATH, +) -> dict[str, Any]: + profile = load_profile(profile_path) + contract = build_contract_report(profile_path) + if contract.get("local_static_contract_verified") is not True: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity static contract is invalid" + ) + + admin_material = SecretStr(secrets.token_urlsafe(24)) + database_material = SecretStr(secrets.token_urlsafe(24)) + initial_material = SecretStr(secrets.token_urlsafe(24)) + rotated_material = SecretStr(secrets.token_urlsafe(24)) + runtime = IsolatedGravitinoRuntime(profile) + forward: provider_metrics._PortForward | None = None + rehearsal: GravitinoIdentityRehearsal | None = None + runtime_observation: dict[str, Any] | None = None + result: dict[str, Any] | None = None + forward_stopped = False + cleanup: dict[str, Any] = { + "namespace_delete_completed": False, + "namespace_absent": False, + "provider_objects_retained": True, + } + try: + runtime_observation = runtime.start( + admin_material=admin_material, + database_material=database_material, + ) + forward = provider_metrics._PortForward( + kubectl="kubectl", + context=profile.cluster.context, + namespace=profile.cluster.rehearsal_namespace, + service=profile.runtime.service, + target_port=profile.runtime.service_port, + ) + forward.start() + rehearsal = GravitinoIdentityRehearsal( + base_url=f"http://127.0.0.1:{forward.local_port}/api", + admin_name=profile.identity.service_admin, + admin_material=admin_material, + ) + result = rehearsal.execute( + profile, + initial_material=initial_material, + rotated_material=rotated_material, + ) + finally: + if rehearsal is not None: + rehearsal.close() + if forward is not None: + forward_stopped = forward.stop() + cleanup = runtime.cleanup() + + if runtime_observation is None or result is None: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity rehearsal did not produce an outcome" + ) + observation = { + "schema": OBSERVATION_SCHEMA, + "observed_at": datetime.now(UTC).isoformat(), + "contract": { + "contract_fingerprint": contract["contract_fingerprint"], + "local_static_contract_verified": True, + }, + "runtime": runtime_observation, + "gravitino": result, + "runtime_checks": { + **cleanup, + "all_port_forwards_stopped": forward_stopped, + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + return build_evidence(observation) + + +def build_validation_report( + *, + profile_path: Path = DEFAULT_PROFILE_PATH, + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + contract = build_contract_report(profile_path) + errors = list(contract["errors"]) + evidence: dict[str, Any] | None = None + try: + value = json.loads(evidence_path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino identity evidence must be an object") + evidence = value + errors.extend(verify_evidence_integrity(evidence)) + observed_contract = _mapping( + _mapping(evidence.get("observation")).get("contract") + ).get("contract_fingerprint") + if observed_contract != contract.get("contract_fingerprint"): + errors.append("Gravitino identity evidence contract fingerprint drift") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + errors.append(f"Gravitino identity evidence is invalid: {type(exc).__name__}") + verified = not errors + return { + "schema": VALIDATION_SCHEMA, + "local_static_contract_verified": contract["local_static_contract_verified"], + "local_gravitino_basic_identity_verified": ( + verified + and evidence is not None + and evidence.get("local_gravitino_basic_identity_verified") is True + ), + "local_gravitino_minimum_privilege_verified": ( + verified + and evidence is not None + and evidence.get("local_gravitino_minimum_privilege_verified") is True + ), + "gravitino_authentication_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "production_identity_verified": False, + "production_ready": False, + "contract_fingerprint": contract["contract_fingerprint"], + "evidence_fingerprint": evidence.get("evidence_fingerprint") if evidence else None, + "errors": errors, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + rehearse = subparsers.add_parser("rehearse") + rehearse.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + rehearse.add_argument("--evidence-out", type=Path, required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_validation_report( + profile_path=args.profile, evidence_path=args.evidence + ) + print(json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not report["errors"] else 1 + if args.command == "verify": + value = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino identity evidence must be an object") + errors = verify_evidence_integrity(value) + print(json.dumps({"verified": not errors, "errors": errors}, indent=2)) + return 0 if not errors else 1 + evidence = run_live_rehearsal(args.profile) + _write_json(args.evidence_out, evidence) + print(json.dumps(evidence, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not evidence["errors"] else 1 + except ( + KeyError, + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + MetadataFabricGravitinoIdentityError, + KeyboardInterrupt, + ) as exc: + print(f"metadata fabric Gravitino identity: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index 590a1819..673ce5fb 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -628,6 +628,26 @@ def _config( ), "Protected OIDC workload identity and authenticated Gravitino access control", ), + RuntimeSpec( + "metadata_gravitino_identity_rehearsal", + "gravitino_identity_rehearsal", + "governed", + "evidence_durable", + "committed local Gravitino Basic bounded identity evidence", + "metadata-platform", + "local_verification_only", + ( + "data_agent/metadata_fabric_gravitino_identity.py", + "scripts/metadata-fabric-gravitino-identity.sh", + ), + ( + ( + "data_agent/metadata_fabric_gravitino_identity.py", + "subprocess.run", + ), + ), + "Protected OIDC workload identity, TLS and production Gravitino catalog", + ), RuntimeSpec( "datalake_monitor", "monitor_loop", diff --git a/data_agent/test_metadata_fabric_gravitino_identity.py b/data_agent/test_metadata_fabric_gravitino_identity.py new file mode 100644 index 00000000..7db34227 --- /dev/null +++ b/data_agent/test_metadata_fabric_gravitino_identity.py @@ -0,0 +1,392 @@ +import base64 +import json +from copy import deepcopy +from pathlib import Path +from urllib.parse import unquote + +import httpx +import yaml +from pydantic import SecretStr + +from data_agent import metadata_fabric_gravitino_identity as identity + + +def _provider_result() -> dict: + return { + "version": "1.3.0", + "configuration": { + "authenticator": "basic", + "access_control_enabled": True, + "idp_extension": "org.apache.gravitino.idp.web.rest.feature", + "built_in_idp_scope": "local_poc_only", + "simple_authenticator_trusted": False, + }, + "authentication": { + "service_admin_status": 200, + "unregistered_principal_status": 401, + "bounded_user_status": 200, + "material_recorded": False, + }, + "principal": { + "name": "gda-metadata-projection", + "granted_name": "gda-metadata-projection", + "roles": ["gda-table-projection"], + "is_service_admin": False, + }, + "role": { + "name": "gda-table-projection", + "readback_name": "gda-table-projection", + "securable_objects": identity._expected_securable_objects(), + }, + "bootstrap": { + "metalake": "gda_identity", + "catalog": "lakehouse", + "schema": "published", + "service_admin_used": True, + "minimum_privilege": False, + }, + "allowed_probe": { + "operation": "CREATE_TABLE", + "resource": "SCHEMA", + "create_status": 200, + "read_status": 200, + "table": "gda_identity_probe", + "readback_table": "gda_identity_probe", + "rotated_readback_table": "gda_identity_probe", + "full_name": "lakehouse.published.gda_identity_probe", + }, + "denied_probe": { + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "catalog": "unauthorized_catalog", + "status": 403, + }, + "login_lifecycle": { + "rotation_mode": "administrator_reset", + "old_after_rotation_status": 401, + "rotated_authentication_status": 200, + "revocation_mode": "idp_user_delete", + "after_revocation_status": 401, + "idp_principal_absent": True, + "material_recorded": False, + }, + "idp_create_response_present": True, + } + + +def _observation() -> dict: + return { + "schema": identity.OBSERVATION_SCHEMA, + "observed_at": "2026-07-28T14:00:00+00:00", + "contract": { + "contract_fingerprint": "a" * 64, + "local_static_contract_verified": True, + }, + "runtime": { + "context": identity.CONTEXT, + "namespace": { + "name": identity.REHEARSAL_NAMESPACE, + "uid": "11111111-1111-4111-8111-111111111111", + }, + "service": { + "name": "gravitino-identity", + "uid": "22222222-2222-4222-8222-222222222222", + "type": "ClusterIP", + }, + "gravitino": { + "kind": "StatefulSet", + "name": "gravitino-identity", + "uid": "33333333-3333-4333-8333-333333333333", + "image": "gda/gravitino:1.3.0-local-arm64", + "service_account": "gravitino-identity", + "service_account_automount_disabled": True, + "ready_replicas": 1, + }, + "postgresql": { + "kind": "StatefulSet", + "name": "gravitino-identity-postgresql", + "uid": "44444444-4444-4444-8444-444444444444", + "image": "postgres:16.10-bookworm", + "service_account": "gravitino-identity-postgresql", + "service_account_automount_disabled": True, + "ready_replicas": 1, + "ephemeral_data_volume": True, + }, + "source_schema_sha256": identity.GRAVITINO_SCHEMA_SHA256, + }, + "gravitino": _provider_result(), + "runtime_checks": { + "namespace_delete_completed": True, + "namespace_absent": True, + "provider_objects_retained": False, + "all_port_forwards_stopped": True, + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + + +def test_static_contract_is_valid_and_explicitly_local(): + report = identity.build_contract_report() + + assert report["local_static_contract_verified"] is True + assert report["errors"] == [] + assert report["authentication"]["authenticator"] == "basic" + assert report["authentication"]["simple_authenticator_trusted"] is False + assert report["gravitino_authentication_verified"] is False + assert report["protected_workload_identity_verified"] is False + assert report["production_ready"] is False + assert all( + not Path(item["path"]).is_absolute() for item in report["files"].values() + ) + + +def test_profile_rejects_broad_privileges_and_claim_overreach(tmp_path): + profile = yaml.safe_load(identity.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["scope"]["role_securable_objects"][1]["privileges"].append( + {"name": "MODIFY_TABLE", "condition": "ALLOW"} + ) + profile["claims"]["production_ready"] = True + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = identity.build_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + assert "profile is invalid" in "\n".join(report["errors"]) + + +def test_profile_rejects_inline_sensitive_material(tmp_path): + profile = yaml.safe_load(identity.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["identity"]["password"] = "must-not-enter-config" + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = identity.build_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + + +def test_valid_evidence_proves_only_local_gravitino_basic_identity(): + evidence = identity.build_evidence(_observation()) + + assert evidence["status"] == "local_gravitino_basic_identity_verified" + assert evidence["local_gravitino_basic_identity_verified"] is True + assert evidence["local_gravitino_minimum_privilege_verified"] is True + assert evidence["local_gravitino_login_rotation_verified"] is True + assert evidence["local_gravitino_revocation_verified"] is True + assert evidence["gravitino_authentication_verified"] is False + assert evidence["protected_workload_identity_verified"] is False + assert evidence["oidc_verified"] is False + assert evidence["production_ready"] is False + assert identity.verify_evidence_integrity(evidence) == [] + + +def test_evidence_blocks_broad_role_failed_denial_and_incomplete_cleanup(): + observation = _observation() + observation["gravitino"]["role"]["securable_objects"][1][ + "privileges" + ].append({"name": "MODIFY_TABLE", "condition": "ALLOW"}) + observation["gravitino"]["denied_probe"]["status"] = 200 + observation["runtime_checks"]["namespace_absent"] = False + + evidence = identity.build_evidence(observation) + + assert evidence["status"] == "blocked" + rendered = "\n".join(evidence["errors"]) + assert "role exceeds" in rendered + assert "catalog-create denial" in rendered + assert "cleanup is incomplete" in rendered + + +def test_evidence_blocks_login_lifecycle_drift_and_sensitive_fields(): + observation = _observation() + observation["gravitino"]["login_lifecycle"][ + "old_after_rotation_status" + ] = 200 + observation["gravitino"]["api_token"] = "must-not-enter-evidence" + + evidence = identity.build_evidence(observation) + + assert evidence["status"] == "blocked" + rendered = "\n".join(evidence["errors"]) + assert "sensitive material" in rendered + assert "rotation or revocation" in rendered + + +def test_evidence_integrity_rejects_claim_and_observation_tampering(): + evidence = identity.build_evidence(_observation()) + tampered = deepcopy(evidence) + tampered["gravitino_authentication_verified"] = True + tampered["observation"]["gravitino"]["denied_probe"]["status"] = 200 + + errors = identity.verify_evidence_integrity(tampered) + + assert any("content or fingerprint drifted" in error for error in errors) + assert any("gravitino_authentication_verified" in error for error in errors) + + +class _GravitinoMock: + def __init__(self) -> None: + self.admin_name = "gda-identity-admin" + self.admin_value = "admin-local-material" + self.user_name = "gda-metadata-projection" + self.user_value: str | None = None + self.user_roles: list[str] = [] + self.role: dict | None = None + self.table: dict | None = None + + @staticmethod + def _response(status: int, payload: dict | None = None) -> httpx.Response: + return ( + httpx.Response(status, json=payload) + if payload is not None + else httpx.Response(status) + ) + + @staticmethod + def _login(request: httpx.Request) -> tuple[str, str] | None: + value = request.headers.get("Authorization", "") + if not value.startswith("Basic "): + return None + decoded = base64.b64decode(value.removeprefix("Basic ")).decode("utf-8") + name, material = decoded.split(":", 1) + return name, material + + def __call__(self, request: httpx.Request) -> httpx.Response: + path = request.url.path.removeprefix("/api/") + method = request.method + body = json.loads(request.content) if request.content else {} + login = self._login(request) + is_admin = login == (self.admin_name, self.admin_value) + is_user = login == (self.user_name, self.user_value) and self.user_value is not None + + if path == "version" and method == "GET": + if not (is_admin or is_user): + return self._response(401, {"code": 1001}) + return self._response(200, {"code": 0, "version": {"version": "1.3.0"}}) + + if not is_admin and not is_user: + return self._response(401, {"code": 1001}) + + if path == "metalakes" and method == "POST" and is_admin: + return self._response(200, {"code": 0, "metalake": body}) + if path == "metalakes/gda_identity/catalogs" and method == "POST": + if is_user: + return self._response(403, {"code": 1002}) + return self._response(200, {"code": 0, "catalog": body}) + if ( + path == "metalakes/gda_identity/catalogs/lakehouse/schemas" + and method == "POST" + and is_admin + ): + return self._response(200, {"code": 0, "schema": body}) + if path == "idp/users" and method == "POST" and is_admin: + self.user_value = body["password"] + return self._response( + 200, {"code": 0, "user": {"name": body["user"]}} + ) + if ( + path == "metalakes/gda_identity/users" + and method == "POST" + and is_admin + ): + return self._response( + 200, {"code": 0, "user": {"name": body["name"], "roles": []}} + ) + if ( + path == "metalakes/gda_identity/roles" + and method == "POST" + and is_admin + ): + self.role = body + return self._response(200, {"code": 0, "role": body}) + if ( + path + == "metalakes/gda_identity/permissions/users/gda-metadata-projection/grant" + and method == "PUT" + and is_admin + ): + self.user_roles = body["roleNames"] + return self._response( + 200, + { + "code": 0, + "user": {"name": self.user_name, "roles": self.user_roles}, + }, + ) + if ( + path == "metalakes/gda_identity/roles/gda-table-projection" + and method == "GET" + and is_admin + ): + role = deepcopy(self.role) + for securable_object in role["securableObjects"]: + securable_object["type"] = securable_object["type"].lower() + for privilege in securable_object["privileges"]: + privilege["name"] = privilege["name"].lower() + privilege["condition"] = privilege["condition"].lower() + return self._response(200, {"code": 0, "role": role}) + if ( + path == "metalakes/gda_identity/catalogs/lakehouse/schemas/published/tables" + and method == "POST" + and is_user + ): + self.table = body + return self._response(200, {"code": 0, "table": body}) + if ( + path + == "metalakes/gda_identity/catalogs/lakehouse/schemas/published/tables/gda_identity_probe" + and method == "GET" + and is_user + ): + return self._response(200, {"code": 0, "table": self.table}) + if path == "idp/users/gda-metadata-projection" and method == "PUT" and is_admin: + self.user_value = body["password"] + return self._response(200, {"code": 0, "user": {"name": self.user_name}}) + if ( + path == "idp/users/gda-metadata-projection" + and method == "DELETE" + and is_admin + ): + self.user_value = None + return self._response(200, {"code": 0, "dropped": True}) + if ( + path == "idp/users/gda-metadata-projection" + and method == "GET" + and is_admin + ): + return self._response(404, {"code": 1003}) + return self._response(404, {"code": 1003, "path": unquote(path)}) + + +def test_http_rehearsal_enforces_basic_role_rotation_revocation_and_denial(): + profile = identity.load_profile() + provider = _GravitinoMock() + rehearsal = identity.GravitinoIdentityRehearsal( + base_url="http://gravitino.test/api", + admin_name=provider.admin_name, + admin_material=SecretStr(provider.admin_value), + transport=httpx.MockTransport(provider), + ) + + try: + result = rehearsal.execute( + profile, + initial_material=SecretStr("initial-local-material"), + rotated_material=SecretStr("rotated-local-material"), + ) + finally: + rehearsal.close() + + assert result["authentication"]["unregistered_principal_status"] == 401 + assert result["allowed_probe"]["create_status"] == 200 + assert result["denied_probe"]["status"] == 403 + assert result["login_lifecycle"]["old_after_rotation_status"] == 401 + assert result["login_lifecycle"]["rotated_authentication_status"] == 200 + assert result["login_lifecycle"]["after_revocation_status"] == 401 + assert result["login_lifecycle"]["idp_principal_absent"] is True + rendered = json.dumps(result) + assert "initial-local-material" not in rendered + assert "rotated-local-material" not in rendered + assert provider.admin_value not in rendered diff --git a/data_agent/test_platform_truth.py b/data_agent/test_platform_truth.py index 103e3ff9..6f5e03fe 100644 --- a/data_agent/test_platform_truth.py +++ b/data_agent/test_platform_truth.py @@ -208,6 +208,11 @@ def test_repository_source_access_and_runtime_baselines_match(): and item["production_role"] == "local_verification_only" for item in static_report["runtime"]["inventory"] ) + assert any( + item["runtime_id"] == "metadata_gravitino_identity_rehearsal" + and item["production_role"] == "local_verification_only" + for item in static_report["runtime"]["inventory"] + ) def test_runtime_report_detects_unregistered_background_mechanism(tmp_path): diff --git a/docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md b/docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md new file mode 100644 index 00000000..964d160a --- /dev/null +++ b/docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md @@ -0,0 +1,67 @@ +# ADR-052: Local Gravitino Basic Bounded Provider Identity + +**Status**: Accepted + +**Date**: 2026-07-28 + +**Decision owners**: Metadata Platform, Data Governance, Security, Platform Architecture + +**Related decisions**: [ADR-006](adr-006-openmetadata-governance-and-active-metadata-platform.md) · [ADR-024](adr-024-dispatch-authorization-evidence.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-047](adr-047-deterministic-metadata-fabric-ingestion-projection.md) · [ADR-051](adr-051-local-openmetadata-bounded-provider-identity.md) + +## Context + +M3-2 used the OpenMetadata bootstrap administrator and an unauthenticated Gravitino `simple` authenticator. M3-5 then proved a bounded OpenMetadata grant, but the Gravitino side still had no honest authentication or authorization evidence. Gravitino `1.3.0` ships a Basic IdP extension, so a separately isolated rehearsal can validate the provider-native identity boundary without mutating the retained M3-2 projection. + +The `simple` authenticator is deliberately excluded: upstream behavior accepts unvalidated usernames and therefore cannot count as authentication. This rehearsal also cannot establish protected Kubernetes workload identity, OIDC federation, TLS, durable catalog conformance or production readiness. + +## Decision + +### 1. Run an isolated, ephemeral provider + +The rehearsal uses Docker Desktop Kubernetes context `docker-desktop` and a pre-existing, checksum-verified Gravitino PostgreSQL schema (`7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df`). It creates only the `gda-metadata-identity` namespace, two single-replica StatefulSets, ClusterIP Services, dedicated ServiceAccounts with token automount disabled, and runtime-generated Kubernetes Secret values. The namespace is deleted before evidence can pass; no Secret value or provider error body enters source, evidence or logs. + +The server is Gravitino `1.3.0` (`gda/gravitino:1.3.0-local-arm64`) with `gravitino.authenticators=basic`, the Basic IdP REST extension and `gravitino.authorization.enable=true`. The provider's relational entity store uses the ephemeral PostgreSQL instance; the probe catalog remains an ephemeral memory catalog and is not a production technical-metadata authority. + +### 2. Keep the administrator as a provisioner only + +The built-in Basic IdP administrator `gda-identity-admin` creates the temporary metalake, catalog, schema, user, role and grant. The exercised principal `gda-metadata-projection` is not a service administrator and receives exactly `gda-table-projection`. + +The role contains only: + +- `USE_CATALOG` on catalog `lakehouse`; +- `USE_SCHEMA` and `CREATE_TABLE` on schema `lakehouse.published`. + +The API client sends the Gravitino 1.3.0 `RoleGrantRequest` field `roleNames` and normalizes provider read-back enum casing at the HTTP boundary. Any extra privilege, role, object or changed scope blocks evidence. + +### 3. Require positive, negative and lifecycle probes + +The rehearsal must observe service-admin authentication (200), an unregistered principal rejection (401), bounded-user authentication (200), table create/read (200/200), and catalog create denial (403). The administrator resets the user password; the old material must return 401 and the replacement must read the probe table. The administrator then deletes the IdP user; the replacement material must return 401 and the IdP lookup must be absent. + +The four local claims are intentionally narrow: `local_gravitino_basic_identity_verified`, `local_gravitino_minimum_privilege_verified`, `local_gravitino_login_rotation_verified` and `local_gravitino_revocation_verified`. They do not imply a protected workload identity or a production provider identity. + +### 4. Preserve the claim boundary + +Evidence fixes `gravitino_authentication_verified=false`, `provider_minimum_privilege_verified=false`, `protected_workload_identity_verified=false`, `oidc_verified=false`, `tls_verified=false`, `production_identity_verified=false` and `production_ready=false`. The local provider result is not combined with M3-5 into a dual-provider production claim: M3-2 still uses the bootstrap administrator, and this rehearsal is isolated from the retained projection and binding ledger. + +## Verification + +The final Docker Desktop rehearsal on 2026-07-28 produced evidence fingerprint `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd` and contract fingerprint `c1c8b08e52c88742fd273378735491626bc4a185f3416746eafa4932374a3704`. + +- Gravitino `1.3.0` and PostgreSQL each reached one ready replica; +- the Basic IdP rejected an unregistered principal with 401 and authenticated the bounded user with 200; +- the bounded role read back exactly `USE_CATALOG`, `USE_SCHEMA` and `CREATE_TABLE` on the declared catalog/schema; +- table create/read returned 200/200 and unauthorized catalog create returned 403; +- password reset invalidated the old material (401), the replacement read the table (200), and IdP deletion invalidated the replacement (401); +- namespace deletion completed, the namespace was absent, all port-forwards stopped and no provider object was retained. + +Focused tests cover strict profile and manifest validation, sensitive-field rejection, exact role scope, positive/negative probes, enum read-back normalization, rotation/revocation, cleanup and evidence tampering. Required CI validates the static contract and runs the focused test module; live Kubernetes evidence remains a local verification artifact. + +## Consequences + +**Positive**: Gravitino now has a real, bounded local authentication and authorization rehearsal with explicit lifecycle and cleanup evidence. The API contract is checked against the actual 1.3.0 DTO instead of relying on a mock-only request shape. + +**Negative**: Basic IdP credentials are locally provisioned by an administrator, the catalog is ephemeral, transport is loopback HTTP, and the provider workload is not authenticated with its Kubernetes ServiceAccount. + +**Mitigation**: retain all production and protected-identity claims as false. The next identity gate must select and validate the protected OIDC/workload exchange, Secret delivery and rotation mechanism for both metadata providers, durable catalog profile, TLS, tenant policy and production conformance. + +**Revisit trigger**: Gravitino version, Basic IdP API, authorization model, catalog backend, cluster identity model or production deployment profile changes. diff --git a/docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json b/docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json new file mode 100644 index 00000000..d65acd41 --- /dev/null +++ b/docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json @@ -0,0 +1,152 @@ +{ + "errors": [], + "evidence_fingerprint": "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd", + "gravitino_authentication_verified": false, + "local_gravitino_basic_identity_verified": true, + "local_gravitino_login_rotation_verified": true, + "local_gravitino_minimum_privilege_verified": true, + "local_gravitino_revocation_verified": true, + "observation": { + "contract": { + "contract_fingerprint": "c1c8b08e52c88742fd273378735491626bc4a185f3416746eafa4932374a3704", + "local_static_contract_verified": true + }, + "gravitino": { + "allowed_probe": { + "create_status": 200, + "full_name": "lakehouse.published.gda_identity_probe", + "operation": "CREATE_TABLE", + "read_status": 200, + "readback_table": "gda_identity_probe", + "resource": "SCHEMA", + "rotated_readback_table": "gda_identity_probe", + "table": "gda_identity_probe" + }, + "authentication": { + "bounded_user_status": 200, + "material_recorded": false, + "service_admin_status": 200, + "unregistered_principal_status": 401 + }, + "bootstrap": { + "catalog": "lakehouse", + "metalake": "gda_identity", + "minimum_privilege": false, + "schema": "published", + "service_admin_used": true + }, + "configuration": { + "access_control_enabled": true, + "authenticator": "basic", + "built_in_idp_scope": "local_poc_only", + "idp_extension": "org.apache.gravitino.idp.web.rest.feature", + "simple_authenticator_trusted": false + }, + "denied_probe": { + "catalog": "unauthorized_catalog", + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "status": 403 + }, + "idp_create_response_present": true, + "login_lifecycle": { + "after_revocation_status": 401, + "idp_principal_absent": true, + "material_recorded": false, + "old_after_rotation_status": 401, + "revocation_mode": "idp_user_delete", + "rotated_authentication_status": 200, + "rotation_mode": "administrator_reset" + }, + "principal": { + "granted_name": "gda-metadata-projection", + "is_service_admin": false, + "name": "gda-metadata-projection", + "roles": [ + "gda-table-projection" + ] + }, + "role": { + "name": "gda-table-projection", + "readback_name": "gda-table-projection", + "securable_objects": [ + { + "fullName": "lakehouse", + "privileges": [ + { + "condition": "ALLOW", + "name": "USE_CATALOG" + } + ], + "type": "CATALOG" + }, + { + "fullName": "lakehouse.published", + "privileges": [ + { + "condition": "ALLOW", + "name": "CREATE_TABLE" + }, + { + "condition": "ALLOW", + "name": "USE_SCHEMA" + } + ], + "type": "SCHEMA" + } + ] + }, + "version": "1.3.0" + }, + "observed_at": "2026-07-28T15:01:18.018090+00:00", + "runtime": { + "context": "docker-desktop", + "gravitino": { + "image": "gda/gravitino:1.3.0-local-arm64", + "kind": "StatefulSet", + "name": "gravitino-identity", + "ready_replicas": 1, + "service_account": "gravitino-identity", + "service_account_automount_disabled": true, + "uid": "7973b9fc-0383-4749-aad2-7d350b9a73a2" + }, + "namespace": { + "name": "gda-metadata-identity", + "uid": "319ca844-b66a-4d70-b256-4df9134601c7" + }, + "postgresql": { + "ephemeral_data_volume": true, + "image": "postgres:16.10-bookworm", + "kind": "StatefulSet", + "name": "gravitino-identity-postgresql", + "ready_replicas": 1, + "service_account": "gravitino-identity-postgresql", + "service_account_automount_disabled": true, + "uid": "ca890e8d-6bfd-4b82-ba64-c8bce0c69831" + }, + "service": { + "name": "gravitino-identity", + "type": "ClusterIP", + "uid": "a80396a3-a9a0-407c-90c1-a3402278a9a6" + }, + "source_schema_sha256": "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df" + }, + "runtime_checks": { + "all_port_forwards_stopped": true, + "kubernetes_service_account_used_for_provider_login": false, + "material_recorded": false, + "namespace_absent": true, + "namespace_delete_completed": true, + "provider_objects_retained": false + }, + "schema": "gda.metadata_fabric_gravitino_identity_observation.v1" + }, + "oidc_verified": false, + "production_identity_verified": false, + "production_ready": false, + "protected_workload_identity_verified": false, + "provider_minimum_privilege_verified": false, + "schema": "gda.metadata_fabric_gravitino_identity_evidence.v1", + "status": "local_gravitino_basic_identity_verified", + "tls_verified": false +} diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index c89e1dd9..0d72c035 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成仅指本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板及离线 activation/release preflight、candidate/registry/provenance/artifact-release/live observation evidence gate、合成 golden slice、定向测试、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。`candidate_validated`、`registry_subject_bound`、本地合成 `provenance_verified`、`ready_for_activation`、`ready_for_staging_apply`、`verified_for_staging_apply` 和本地 live collection 都不等于真实镜像已 attested 或 staging 已部署；真实 IAM/OIDC 与 service token 生命周期、首次 GHCR publish/verify、真实 provenance artifact verify、registry-backed live staging revision、worker/callback 扩容运行、golden slice staging 运行链、受保护 release/live evidence provenance、独立 DolphinScheduler metadata PostgreSQL 和真实数据终局证据仍属于 4.7 后续切片。 -### 4.8 Metadata Fabric Bridge M1 + M2 + M3-5（本地 OpenMetadata 受限身份已验证） +### 4.8 Metadata Fabric Bridge M1 + M2 + M3-6（本地双 provider 受限身份已分别验证） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -225,8 +225,9 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 20. [ADR-049](architecture-decisions/adr-049-tenant-scoped-metadata-fabric-binding-ledger.md) 已新增 migration 097 与 `PlatformGateway` binding commit/read：真实 provider refs、target/source/definition version、execution-plan、精确 PolicyDecision、独立 Approval 和 provider evidence 必须在同一 tenant 下完整匹配后才可追加。空临时 PostgreSQL 首次提交为 `created=true`、第二次精确 replay 为 `created=false`，FORCE RLS、跨租户不可见和 gateway 无 UPDATE/DELETE 均通过；binding UUID 为 `9580cd65-9fd9-5216-90a5-1fd6837e6cfb`，record SHA 为 `19bdbddedc27d2ed8a35119e8f065a47a02345f9bbd3a51075856cb9587f4176`，evidence SHA 为 `518bfed363aba34e539ada19ea1dc708bacc9eba6578ccab165d11bccfc05223`。M3-3 不调用 provider、不写 legacy，也不覆盖含 synthetic UUID 的既有 Resource。 21. [ADR-050](architecture-decisions/adr-050-idempotent-openlineage-http-delivery.md) 已新增 migration 098 与 tenant-scoped lineage outbox；Gateway 只有在 M3-3 binding、execution-plan 和完整 M3-1 source plan 精确匹配后才可 enqueue。真实 loopback HTTP 演练让接收端先提交事件再返回 503，第二次以同一 `Idempotency-Key` 重发并返回 duplicate 200；共 2 个 wire requests、1 次唯一接受、最终 2 attempts/delivered，完成项不再 claim。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `8fa87a34a39b900df0673f11d0301c9f5155ce64ff9502125478ec59a3f0fdb6`。该结论是本地 `at_least_once_with_receiver_idempotency`，不是网络 exactly-once 或生产 receiver 证明。 22. [ADR-051](architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md) 已在 OpenMetadata `1.13.1` 创建临时非管理员 bot，effective roles 精确为 provider 强制 `DefaultBotRole` 加项目专用 role；项目新增 policy 仅允许 `table/Create`。bot 创建/read-back 临时 table 分别返回 201/200，创建 policy 返回 403；JWT 轮换后旧值返回 401，新值返回 200，吊销后新值返回 401。table、bot、user、role、policy 与 denial probe 最终均不存在，evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。该 scoped local 结论不包含 Gravitino、OIDC、Kubernetes-to-provider identity exchange 或生产 credential delivery。 +23. [ADR-052](architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md) 已在隔离 Gravitino `1.3.0` namespace 启用 Basic IdP 与 authorization，bounded user 只获得 `lakehouse` 的 `USE_CATALOG` 以及 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，越权 catalog create 为 403，密码轮换使旧值返回 401，IdP 用户删除使替换值返回 401。临时 metalake/catalog/schema/table/user/role、namespace 与 loopback port-forward 均已清理，证据 SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。这只是 Gravitino Basic local POC；probe catalog 使用 memory backend，不包含 OIDC、TLS、Kubernetes-to-provider identity exchange、持久生产 catalog 或双 provider production identity。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上新增的 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝。M3-2 ingestion 仍使用 bootstrap admin，Gravitino 未认证且 catalog backend 为 memory，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、持久 catalog、TLS、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、两项 production gate 和 `production_ready` 仍为 `false`。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理。M3-2 ingestion 仍使用 bootstrap admin，Gravitino M3-6 probe catalog 使用 memory backend，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、TLS、持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、两项 production gate 和 `production_ready` 仍为 `false`。 ## 5. 重新评估条件 diff --git a/docs/roadmap.md b/docs/roadmap.md index 218e5394..9535f9d2 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -417,7 +417,7 @@ AR-0 Architecture / Schema / Runtime Truth - Web、DolphinScheduler、OpenMetadata、Gravitino、worker 和 bridge 重启后不丢版本、产品、血缘、PlatformRun correlation 或审批状态。 - DataOps 状态可从 DolphinScheduler/OpenMetadata/Gravitino/GDA evidence 恢复；任何 release/deployment 都能从事件、Artifact、评测、策略、incident 和 rollback pointer 重放。Temporal durable recovery 在 AR-5/AR-7 单独验收。 -当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 又以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401。这个 `local_openmetadata_minimum_privilege_verified=true` 明确限定在 provider 强制 `DefaultBotRole` 之上的项目新增 grant；M3-2 ingestion 仍使用 bootstrap admin，Gravitino 仍未认证且使用 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、持久 catalog、生产 ingestion/conformance、生产 lineage receiver 与 `production_ready` 仍为 false。 +当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver 与 `production_ready` 仍为 false。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 4dd55196..59de5474 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,9 +2,9 @@ 日期：2026-07-28 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery 与 M3-5 local OpenMetadata bounded identity 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity 与 M3-6 local Gravitino Basic bounded identity 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-provider-identity` +适用分支：`feat/ar1-metadata-fabric-gravitino-identity` ## 判定规则 @@ -25,7 +25,7 @@ | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC | GDA ledger 管身份与版本绑定；旧行只有在 tenant、authority identity、checksum 和 version evidence 完整时才可形成 eligible plan；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway 已验证 -> 生产切换待验收 | -| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory catalog/evidence 不得冒充生产持久技术权威 | Metadata Platform | AR-1 M1/M2 + M3-3 local ledger 已验证 -> 认证持久 catalog/生产 conformance 待执行 | +| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger；M3-6 又在隔离 Gravitino Basic IdP 中验证 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理 | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory catalog/evidence 不得冒充生产持久技术权威；M3-6 的 Basic IdP 和 loopback HTTP 也不是生产身份 | Metadata Platform | AR-1 M1/M2 + M3-6 local identity 已验证 -> 认证持久 catalog/受保护 workload identity/生产 conformance 待执行 | | 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver；M3-5 已验证临时非管理员 bot 的 scoped `table/Create` grant、policy-create 拒绝及 JWT 轮换/吊销 | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection/provider evidence、binding ledger、lineage outbox/receipt 与 OpenLineage event | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；均不反写 ResourceVersion | Governance | AR-1 M1/M2 + M3-5 local bounded identity 已验证 -> protected identity ingestion/生产持久 binding/受保护 production receiver 待执行 | | 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion 与 event checksum 证据完整的旧记录可形成 eligible plan；目录图只作可重建投影 | Data Platform | AR-1 gateway 已验证 -> adapter 待接入 | | Definition | `gda_control.platform_definition_version` 已绑定 definition ResourceVersion、完整逻辑 hash 和原子 gateway registration；3.4.2 adapter 可编译、创建并上线 provider DAG；binding 已以 append-only `execution_plan` Artifact 持久化并可按 tenant + artifact UUID 读取，旧 workflow/template/YAML 仍在写入 | 编辑器状态、DolphinScheduler DAG/definition | 旧 workflow 必须规范化并完整 hash 后才可形成 PlatformDefinitionVersion；provider binding 作为 ExecutionPlanArtifact/evidence，不可反写 definition | DataOps | AR-1 binding persistence 代码已验证 -> staging 调用链待验收 | @@ -34,7 +34,7 @@ | 事件交付 | Standards outbox 已数据库耐久；`gda_control.platform_command_outbox` 已为 DolphinScheduler dispatch/reconcile 提供 tenant RLS、lease claim、幂等 callback、薄 consumer library 和 managed worker process；其他 WebSocket/bot/feedback 多为 best effort | command delivery status、消费者 claim、worker status JSON、WebSocket 消息 | command/event 与源事实同事务入 outbox，幂等 consumer 交付；worker status、outbox 状态、缓存或 socket 都不是 Run/业务权威 | Platform/Integrations | AR-1 command delivery/worker 代码已验证 -> staging worker/callback 待部署 | | 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；OpenMetadata 与 UI 只作可重建发现投影；旧结果缺稳定版本和证据时不得升级为终局依据 | Governance/DataOps | AR-1 最小成功证据已验证 -> 真实规则/staging 待接入 | | 标准与语义定义 | `std_*`、semantic registry 和 YAML 共同存在，生命周期未统一 | prompt/context、搜索索引 | 版本化 Standard/SemanticDefinition 经审批后为权威；Agent context 只消费批准版本 | Governance | AR-1 -> AR-3 | -| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5 本地 OpenMetadata bot 已验证 scoped project grant、越权拒绝和 JWT 轮换/吊销，但由 bootstrap admin 临时 provision，Gravitino 未认证 | session/cache、前端菜单权限、本地 provider identity evidence | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；完成受保护 OIDC/IAM provisioning、双 provider 最小权限、secret delivery 和轮换/吊销后才能生产切换 | Security | AR-1 local OpenMetadata identity 已验证 -> protected 双 provider IAM 待验收 | +| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5 本地 OpenMetadata bot 和 M3-6 隔离 Gravitino Basic user 分别验证 scoped grant、越权拒绝和 credential rotation/revocation，但均由本地管理员临时 provision，Gravitino 使用 loopback HTTP | session/cache、前端菜单权限、本地 provider identity evidence | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；完成受保护 OIDC/IAM provisioning、双 provider 最小权限、secret delivery、TLS 和轮换/吊销后才能生产切换 | Security | AR-1 local provider identities 已分别验证 -> protected 双 provider IAM 待验收 | | GIS 服务定义与 active revision | Martin、REST/MVT/STAC endpoints 和配置直接暴露 | Ingress、tile cache、客户端图层 | GIS Service Control Plane 管 Service/Layer/Style/TMS/DeploymentRevision；provider/Gateway 仅执行 | GIS Platform | AR-4 | | 缓存与进度 | Redis、进程内 dict/task map | UI progress、tile/context cache | 永不作为资产、Run、workflow 或产品权威；丢失后必须从 ledger/provider 重建 | SRE/Platform | 持续约束 | | Agent/Prompt/Model bundle | 多个 registry 与 YAML 存在，尚无统一 deployment revision | trace、eval dashboard | AgentSpecBundle + EvaluationBinding + DeploymentRevision；只消费已发布 DataProductVersion | AgentOps | AR-5 | @@ -56,7 +56,7 @@ 11. `platform_command_outbox` 只拥有投递状态；callback 只触发 reconcile，不能把 provider payload 直接写成 PlatformRun 状态或平台终局。 12. QualityResult evaluator 必须是 workload，且成功终局中的 evaluator 不能等于 Run workload；该代码级职责分离不替代生产 IAM。 13. `candidate_validated`、`registry_subject_bound`、GitHub provenance action 成功、CI artifact、离线 preflight、未独立 attested 的 live observation JSON 或人工批准都不能单独授权 production；缺少同一 source revision 的 OCI subject 独立验证、registry/live revision/identity/health/golden-slice 绑定及受保护 provenance 时，promotion 必须失败。 -14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销。本地 bootstrap provisioner、未认证 Gravitino、memory catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 catalog/binding、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 +14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销；M3-6 只证明隔离 Gravitino Basic user 的 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理。本地 bootstrap provisioner、Basic IdP、loopback HTTP、memory catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 catalog/binding、TLS、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 ## 已建立的 AR-0/AR-1 entry 证据 diff --git a/k8s/metadata-fabric-gravitino-identity/kustomization.yaml b/k8s/metadata-fabric-gravitino-identity/kustomization.yaml new file mode 100644 index 00000000..fc965874 --- /dev/null +++ b/k8s/metadata-fabric-gravitino-identity/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: gda-metadata-identity +resources: + - namespace.yaml + - runtime.yaml +labels: + - pairs: + app.kubernetes.io/part-of: gda-metadata-fabric-gravitino-identity + gda.openai.com/environment: identity-rehearsal + includeSelectors: false diff --git a/k8s/metadata-fabric-gravitino-identity/namespace.yaml b/k8s/metadata-fabric-gravitino-identity/namespace.yaml new file mode 100644 index 00000000..245f0b39 --- /dev/null +++ b/k8s/metadata-fabric-gravitino-identity/namespace.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gda-metadata-identity + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gravitino-identity-rehearsal + namespace: gda-metadata-identity +spec: + hard: + requests.cpu: "2" + requests.memory: 3Gi + limits.cpu: "4" + limits.memory: 6Gi + pods: "4" + secrets: "4" + configmaps: "4" diff --git a/k8s/metadata-fabric-gravitino-identity/runtime.yaml b/k8s/metadata-fabric-gravitino-identity/runtime.yaml new file mode 100644 index 00000000..00dbe47d --- /dev/null +++ b/k8s/metadata-fabric-gravitino-identity/runtime.yaml @@ -0,0 +1,259 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gravitino-identity-postgresql + namespace: gda-metadata-identity +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gravitino-identity + namespace: gda-metadata-identity +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gravitino-identity-config + namespace: gda-metadata-identity +data: + gravitino.conf: | + gravitino.server.shutdown.timeout = 30000 + gravitino.server.webserver.host = 0.0.0.0 + gravitino.server.webserver.httpPort = 8090 + gravitino.server.health.entityStore.probeTimeoutMs = 5000 + gravitino.entity.store = relational + gravitino.entity.store.relational = JDBCBackend + gravitino.entity.store.relational.jdbcUrl = jdbc:postgresql://gravitino-identity-postgresql:5432/gravitino + gravitino.entity.store.relational.jdbcDriver = org.postgresql.Driver + gravitino.entity.store.relational.jdbcUser = gravitino + gravitino.authenticators = basic + gravitino.server.rest.extensionPackages = org.apache.gravitino.idp.web.rest.feature + gravitino.authorization.enable = true + gravitino.authorization.serviceAdmins = gda-identity-admin + gravitino.cache.enabled = false + gravitino.auxService.names = +--- +apiVersion: v1 +kind: Service +metadata: + name: gravitino-identity-postgresql + namespace: gda-metadata-identity +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gravitino-identity-postgresql + ports: + - name: postgresql + port: 5432 + targetPort: postgresql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gravitino-identity-postgresql + namespace: gda-metadata-identity +spec: + serviceName: gravitino-identity-postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gravitino-identity-postgresql + template: + metadata: + labels: + app.kubernetes.io/name: gravitino-identity-postgresql + app.kubernetes.io/component: database + spec: + serviceAccountName: gravitino-identity-postgresql + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: postgresql + image: postgres:16.10-bookworm + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_USER + value: gravitino + - name: POSTGRES_DB + value: gravitino + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: gravitino-identity-runtime + key: database-password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgresql + containerPort: 5432 + startupProbe: + exec: + command: ["pg_isready", "-U", "gravitino", "-d", "gravitino"] + periodSeconds: 3 + failureThreshold: 60 + readinessProbe: + exec: + command: ["pg_isready", "-U", "gravitino", "-d", "gravitino"] + periodSeconds: 5 + failureThreshold: 12 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: runtime + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + - name: schema + mountPath: /docker-entrypoint-initdb.d/001-schema.sql + subPath: 001-schema.sql + readOnly: true + volumes: + - name: data + emptyDir: {} + - name: runtime + emptyDir: {} + - name: tmp + emptyDir: {} + - name: schema + configMap: + name: gravitino-identity-schema +--- +apiVersion: v1 +kind: Service +metadata: + name: gravitino-identity + namespace: gda-metadata-identity +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gravitino-identity + ports: + - name: http + port: 8090 + targetPort: http +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gravitino-identity + namespace: gda-metadata-identity +spec: + serviceName: gravitino-identity + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gravitino-identity + template: + metadata: + labels: + app.kubernetes.io/name: gravitino-identity + app.kubernetes.io/component: technical-metadata + spec: + serviceAccountName: gravitino-identity + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: gravitino + image: gda/gravitino:1.3.0-local-arm64 + imagePullPolicy: Never + command: ["/bin/bash", "-ec"] + args: + - | + install -d -m 0700 /var/run/gravitino/conf + install -m 0600 /etc/gravitino/gravitino.conf /var/run/gravitino/conf/gravitino.conf + install -m 0444 /opt/gravitino/conf/gravitino-env.sh /var/run/gravitino/conf/gravitino-env.sh + printf '\ngravitino.entity.store.relational.jdbcPassword = ' >> /var/run/gravitino/conf/gravitino.conf + cat /var/run/secrets/gravitino/database-password >> /var/run/gravitino/conf/gravitino.conf + printf '\n' >> /var/run/gravitino/conf/gravitino.conf + exec /opt/gravitino/bin/gravitino.sh --config /var/run/gravitino/conf run + env: + - name: GRAVITINO_MEM + value: -Xms384m -Xmx1024m -XX:MaxMetaspaceSize=384m + - name: GRAVITINO_INITIAL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: gravitino-identity-runtime + key: admin-password + ports: + - name: http + containerPort: 8090 + startupProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 5 + failureThreshold: 120 + readinessProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 10 + failureThreshold: 12 + resources: + requests: + cpu: 250m + memory: 768Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /etc/gravitino + readOnly: true + - name: secrets + mountPath: /var/run/secrets/gravitino + readOnly: true + - name: runtime + mountPath: /var/run/gravitino + - name: logs + mountPath: /opt/gravitino/logs + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: gravitino-identity-config + - name: secrets + secret: + secretName: gravitino-identity-runtime + defaultMode: 0400 + - name: runtime + emptyDir: {} + - name: logs + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/scripts/metadata-fabric-gravitino-identity.sh b/scripts/metadata-fabric-gravitino-identity.sh new file mode 100755 index 00000000..fdd9f92a --- /dev/null +++ b/scripts/metadata-fabric-gravitino-identity.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_gravitino_identity "$@"