From cad23db033549d383f7f492d1baa047568271121 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:36:59 +0530 Subject: [PATCH 01/81] feat: add a docker compose deployment of the whole stack [OpenAgriNet/network-adapter#4] One compose file brings up the registry, the discovery service and the three adapters, with every config under config/. Intended for trying the network end to end on one machine, not for production: the adapter keys live in a config file and nothing runs TLS. Two things are deliberately absent, because they are the user's own: the provider API run it locally and expose it with ngrok. No config here names it -- the provider adapter reads its base URL from the registry at request time, so moving to a new tunnel is a registry edit and nothing more. its registry rows the Participant and ProviderSchema are created by hand, since the base URL is that tunnel. bin/setup.py does only what cannot be done by hand: it generates the three adapter keypairs, registers the three adapter identities, and renders the adapter configs with the key osid the registry assigned. Those three rows have to exist before any adapter can sign or verify anything, and the osid is only knowable after the write. The images build from remote git contexts, so no sibling checkout is needed. ADAPTER_SRC points at a feature branch because the three OAN plugins are not on the adapter's default branch yet. config/mappings/ carries the published mapping to read and to fork, and is not served from here. The registry holds a full URL and the adapter fetches it verbatim, so a mapping has to be published before it can be tested -- serving the local copy would prove the file works and prove nothing about the file anyone else reads. No Makefile: the four commands are in the README. --- docker-deployment/.env.example | 85 + docker-deployment/.gitignore | 14 + docker-deployment/README.md | 280 ++ docker-deployment/bin/setup.py | 313 +++ .../config/adapters/exp.yaml.tmpl | 66 + .../config/adapters/network.yaml.tmpl | 69 + .../config/adapters/provider.yaml.tmpl | 97 + .../config/adapters/routing-exp.yaml | 25 + .../config/adapters/routing-network.yaml | 14 + .../config/discovery/instance.yaml.example | 42 + .../weather-observation.select.yaml | 247 ++ .../config/registry/imports/realm-export.json | 2321 +++++++++++++++++ .../config/registry/schemas/Participant.json | 536 ++++ .../registry/schemas/ProviderSchema.json | 83 + .../registry/schemas/SchemaRegistry.json | 36 + docker-deployment/docker-compose.yml | 233 ++ 16 files changed, 4461 insertions(+) create mode 100644 docker-deployment/.env.example create mode 100644 docker-deployment/.gitignore create mode 100644 docker-deployment/README.md create mode 100755 docker-deployment/bin/setup.py create mode 100644 docker-deployment/config/adapters/exp.yaml.tmpl create mode 100644 docker-deployment/config/adapters/network.yaml.tmpl create mode 100644 docker-deployment/config/adapters/provider.yaml.tmpl create mode 100644 docker-deployment/config/adapters/routing-exp.yaml create mode 100644 docker-deployment/config/adapters/routing-network.yaml create mode 100644 docker-deployment/config/discovery/instance.yaml.example create mode 100644 docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml create mode 100644 docker-deployment/config/registry/imports/realm-export.json create mode 100644 docker-deployment/config/registry/schemas/Participant.json create mode 100644 docker-deployment/config/registry/schemas/ProviderSchema.json create mode 100644 docker-deployment/config/registry/schemas/SchemaRegistry.json create mode 100644 docker-deployment/docker-compose.yml diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example new file mode 100644 index 0000000..b698380 --- /dev/null +++ b/docker-deployment/.env.example @@ -0,0 +1,85 @@ +# Copy to .env and read it through once. Nothing here is a real secret: every +# value is local to this stack, and the adapter keypairs are generated by +# bin/setup.py into keys/keys.json rather than being written here. + +# ---- where the images come from -------------------------------------------- +# A remote git context, so no sibling checkout is needed: docker clones it. +# The fragment after # is the ref. +# +# It points at the feature branch because the three OAN plugins -- oanregistry, +# jsonmapper and the weather provider step -- are not on the default branch +# yet. Change it to #development once that has merged. +# +# Working on the adapter? Point this at a local path instead: +# ADAPTER_SRC=../../network-adapter +ADAPTER_SRC=https://github.com/OpenAgriNet/network-adapter.git#feat/41-oan-adapter-plugins +DISCOVERY_SRC=https://github.com/OpenAgriNet/discovery-service.git + +# Tags for the images built from the above. Set these to a published image to +# skip building altogether. +ADAPTER_IMAGE=oan/network-adapter:local +DISCOVERY_IMAGE=oan/discovery-service:local + +# ---- ports (host side) ----------------------------------------------------- +REGISTRY_PORT=8081 +KEYCLOAK_PORT=8080 +KEYCLOAK_ADMIN_PORT=9990 +DISCOVERY_PORT=8090 +PROVIDER_ADAPTER_PORT=9200 +NETWORK_ADAPTER_PORT=9201 +EXP_ADAPTER_PORT=9202 + +# ---- registry -------------------------------------------------------------- +REGISTRY_VERSION=v2.0.0 +POSTGRES_DB=registry +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +KEYCLOAK_REALM=sunbird-rc +KEYCLOAK_ADMIN_USER=admin +KEYCLOAK_ADMIN_PASSWORD=admin +KEYCLOAK_SECRET=1b46c0f2-6a1c-4a2f-8b0f-2b6b30bcd2b2 +KEYCLOAK_ADMIN_CLIENT_ID=admin-api +KEYCLOAK_CLIENT_ID=registry-frontend +REGISTRY_DEFAULT_USER_PASSWORD=abcd@123 +REGISTRY_USER=no-user +REGISTRY_PASSWORD=no-user-password + +# ---- discovery ------------------------------------------------------------- +APP_NETWORK_ID=local-network +BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml + +# ---- the three adapter identities ------------------------------------------ +# bin/setup.py registers exactly these three in the registry and generates a +# keypair for each. A participant id IS the network identity -- what goes on +# the wire as context.bapId / bppId -- so the registry requires it to be +# hostname-shaped. These names are never resolved by DNS: routing between the +# adapters is the router plugin's config, which uses the compose service names. +EXP_SUBSCRIBER_ID=exp.oan.local +NETWORK_SUBSCRIBER_ID=network.oan.local +PROVIDER_SUBSCRIBER_ID=provider.oan.local + +# ---- YOUR provider --------------------------------------------------------- +# You create these two registry rows by hand -- see README.md. setup.py does +# NOT create them, because the base URL is your own ngrok tunnel. +# +# These two values must MATCH the rows you create: they are rendered into the +# provider adapter's config as the binding key it answers to. A mismatch means +# the adapter passes your request straight through and you get a bare ACK with +# no on_select, which looks like nothing happened. +PROVIDER_PARTICIPANT_ID=my-weather-api +PROVIDER_CAPABILITY=openagrinet:WeatherObservation + +# The mapping the provider adapter fetches, request and response in one file. +# The registry row holds the full URL and the adapter fetches it verbatim. +# +# This defaults to the published copy, which is the same file you will find in +# config/mappings/ -- that copy is there to read and to fork, not to be served +# from here. A mapping has to be published somewhere the adapter can reach +# before it can be tested, so what this stack exercises is what consumers +# actually fetch. Serving the local copy would prove the file works and prove +# nothing about the file anyone else reads. +# +# Forking it? Publish your copy (a raw GitHub URL is fine) and put that URL in +# the ProviderSchema row you create. This variable is only the default the +# README's example curl uses. +MAPPING_URL=https://raw.githubusercontent.com/ameersohel45/oan-mappings/main/mausamgram/weather-observation.select.yaml diff --git a/docker-deployment/.gitignore b/docker-deployment/.gitignore new file mode 100644 index 0000000..e50986d --- /dev/null +++ b/docker-deployment/.gitignore @@ -0,0 +1,14 @@ +# Real values, and the ports/refs one machine happens to use. +.env + +# Private keys. Generated by bin/setup.py, never committed. +keys/ + +# Rendered from the .tmpl files beside them, and they carry the private key +# material that keys/ holds. The templates are the tracked source. +config/adapters/exp.yaml +config/adapters/network.yaml +config/adapters/provider.yaml + +# A local override for the discovery service, if you make one. +config/discovery/instance.yaml diff --git a/docker-deployment/README.md b/docker-deployment/README.md new file mode 100644 index 0000000..e3d9de2 --- /dev/null +++ b/docker-deployment/README.md @@ -0,0 +1,280 @@ +# OAN stack, on Docker Compose + +The whole OpenAgriNet stack on one machine: the registry, the discovery +service, and the three adapters. One compose file, one config folder. + +This is for trying the network end to end on your own laptop. It is not a +production deployment — the keys live in a config file and nothing runs TLS. + +## What is here, and what is not + +Running here: + +- **registry** — SunbirdRC, plus its Postgres and Keycloak. Holds who is on + the network, their public keys, and which upstream API answers which + capability. +- **discovery** — catalogue search, plus its own Postgres. +- **three adapters** — experience, network and provider. Same image, three + configs. + +Deliberately **not** here: + +- **your provider API.** You run it yourself and expose it with ngrok. The + provider adapter never has its address in a config file — it reads it from + the registry at request time, so swapping tunnels is a registry edit. +- **the provider's registry rows.** You create those two by hand, because the + base URL is your tunnel. `bin/setup.py` registers only the three adapters. + +## Before you start + +- Docker with Compose v2 +- Python 3 and the `cryptography` package — `pip install cryptography` +- [ngrok](https://ngrok.com/) or any other way to give your local API a public + https URL + +## Bring it up + +```sh +cp .env.example .env +``` + +Read `.env` before going on. Two things in it matter: + +- `ADAPTER_SRC` points at a **feature branch**, because the three OAN plugins + are not on the adapter's default branch yet. +- `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` have to match the + registry rows you create further down. + +Then: + +```sh +# 1. registry and discovery. The adapters will fail to start for now -- +# their configs do not exist yet, which step 2 fixes. +docker compose up -d + +# 2. generate the adapter keypairs, register the three adapter identities, +# render the three adapter configs +python3 bin/setup.py + +# 3. now the adapters have configs to read +docker compose up -d +``` + +The first run builds the adapter and discovery images from source, so give it +a few minutes. + +Check it: + +```sh +docker compose ps +curl -s -X POST http://localhost:8081/api/v1/Participant/search \ + -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool +``` + +Three participants, one per adapter. That is what `setup.py` seeded. + +## Register your provider + +Two rows. Both by hand, and both need a token. + +Start your API and expose it: + +```sh +ngrok http 9100 +``` + +Take the `https://` URL ngrok prints. Get a token: + +```sh +TOKEN=$(curl -s -X POST \ + "http://localhost:8080/auth/realms/sunbird-rc/protocol/openid-connect/token" \ + -H 'X-Forwarded-Host: keycloak:8080' -H 'X-Forwarded-Proto: http' \ + -d 'client_id=registry-frontend' -d 'grant_type=password' \ + -d 'username=no-user' -d 'password=no-user-password' \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') +``` + +Those two `X-Forwarded-*` headers are not optional. Keycloak runs behind +`PROXY_ADDRESS_FORWARDING` here, and without them it answers with an empty +body. + +**Row one — the API itself.** Type `upstream`: it has no role and no keys, +because it has never heard of Beckn. + +```sh +curl -s -X POST http://localhost:8081/api/v1/Participant \ + -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ + -d '{ + "participantId": "my-weather-api", + "name": "My weather API", + "type": "upstream", + "status": "active", + "baseUrl": "https://YOUR-NGROK-SUBDOMAIN.ngrok-free.app", + "auth": { "scheme": "none" } + }' +``` + +**Row two — which capability it answers, and how to call it.** + +```sh +curl -s -X POST http://localhost:8081/api/v1/ProviderSchema \ + -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ + -d '{ + "bindingKey": "my-weather-api|openagrinet:WeatherObservation", + "participantId": "my-weather-api", + "capabilityCode": "openagrinet:WeatherObservation", + "status": "active", + "actions": [{ + "action": "select", + "method": "GET", + "path": "/get-daily", + "mappings": "https://raw.githubusercontent.com/ameersohel45/oan-mappings/main/mausamgram/weather-observation.select.yaml", + "timeoutMs": 15000, + "retryMax": 2, + "status": "active" + }] + }' +``` + +Things worth knowing about these two calls: + +- **No `{"Participant": {...}}` wrapper.** The registry takes the record + itself. A wrapper comes back as `extraneous key [Participant] is not + permitted`. +- **`bindingKey` is `participantId|capabilityCode`.** It has to match what + `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` were set to in `.env` + when you ran `setup.py`, because that is the key the provider adapter was + configured to answer to. +- **`path` must start with one `/` and contain no empty segment.** The schema + refuses `//get-daily`, and so does the adapter. +- **This registry is append-only.** There is no update, delete is soft, and a + soft-deleted id keeps the unique index — so an id can never be reused. Got a + row wrong? Pick a new id. + +## Test it end to end + +Replace `my-weather-api` if you used a different id, and point the coordinates +at wherever your API has data. + +```sh +curl -s -X POST http://localhost:9202/beckn/select \ + -H 'Content-Type: application/json' \ + -d '{ + "context": { + "version": "2.0.0", "action": "select", + "networkId": "local-network", + "bapId": "exp.oan.local", "bapUri": "http://exp-adapter:9202/beckn", + "bppId": "provider.oan.local", "bppUri": "http://provider-adapter:9200/beckn", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-09-02T06:12:01.330Z" + }, + "message": { "contract": { "commitments": [ { + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [ { + "id": "res:point-forecast", + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": ["Weather"], + "location": { "type": "Point", "coordinates": [73.7898, 19.9975] } + } + } ], + "offer": { + "id": "offer:open-data", + "resourceIds": ["res:point-forecast"], + "provider": { "id": "my-weather-api", + "descriptor": { "code": "MY-API-01", "name": "My weather API" } } + } + } ] } } + }' | python3 -m json.tool +``` + +You should get an `on_select` back, with one resource per forecast day. + +The experience adapter is the only one that takes an unsigned request — the +experience app is inside the trust boundary, so there is no network signature +to check. That is what makes this testable with a plain curl. + +## The layout + +``` +docker-compose.yml the whole stack +.env.example copy to .env +bin/setup.py keys, the three adapter rows, the adapter configs +config/ + adapters/ + exp.yaml.tmpl templates. setup.py renders these to .yaml, + network.yaml.tmpl filling in the keys it generated. The rendered + provider.yaml.tmpl files hold private keys and are gitignored. + routing-exp.yaml which action goes to which adapter + routing-network.yaml + registry/ + schemas/ Participant, ProviderSchema, SchemaRegistry. + Read at startup -- a change needs the registry + service restarted. + imports/ the Keycloak realm + discovery/ + instance.yaml.example optional override; see the compose file + mappings/ + mausamgram/ the request and response transformation +``` + +## About the mapping file + +`config/mappings/` holds the same file the published mappings repository +serves. It is there to **read and to fork** — not to be served from here. + +The registry row holds a full URL and the adapter fetches it verbatim, so a +mapping has to be published somewhere the adapter can reach before it can be +tested. What this stack exercises is therefore what consumers actually fetch. +Serving the local copy would prove the file works and prove nothing about the +file anyone else reads. + +To change the mapping: fork it, publish your copy anywhere that serves raw +text over https, and put that URL in the `mappings` field of your +ProviderSchema row. + +The adapter caches a mapping for `cacheTTL` (one minute, in the adapter +config) and GitHub's raw CDN caches for about five, so give an edit a few +minutes to show up. + +## When it does not work + +**`{"status":"ACK"}` and no `on_select`.** The provider adapter did not +recognise the request as its own, so it passed it through. The binding key in +your `ProviderSchema` row does not match what the adapter is configured for. +Compare the row against `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` in +`.env`, and re-run `bin/setup.py` if you change them. + +**The adapters restart in a loop on the first `up`.** Expected before +`bin/setup.py` has run — there is no `config/adapters/*.yaml` yet. + +**`setup.py` says the registry did not come up.** Check `docker compose ps`. +The registry waits on Keycloak, which waits on Postgres, so a cold start takes +a minute or two. + +**`setup.py` says a participant is registered with a different key.** You have +a `keys/keys.json` that no longer matches the registry. Restore the old one, or +pick new `*_SUBSCRIBER_ID` values in `.env` — the old ids cannot be reused. + +**A build fails on `go mod download`, or the adapter cannot fetch the mapping, +with "network is unreachable".** Your machine advertises IPv6 but cannot route +it. Add this to the adapter service in the compose file: + +```yaml + sysctls: + - net.ipv6.conf.all.disable_ipv6=1 +``` + +and, if the build itself is what fails, `network: host` under its `build:`. + +## Starting over + +```sh +docker compose down -v # -v also deletes the registry and discovery data +rm -rf keys config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml +``` + +Then start again from `docker compose up -d`. New keys mean new identities, so +the provider rows have to be created again too. diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py new file mode 100755 index 0000000..e1ecf07 --- /dev/null +++ b/docker-deployment/bin/setup.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Prepare the stack: generate the adapter keypairs, register the three adapter +identities, render the adapter configs. + + python3 bin/setup.py + +Safe to re-run. Keys are generated once and reused from keys/keys.json, so the +identities already in the registry stay valid; participants that exist are left +alone rather than recreated, because this registry's delete is soft and holds +the unique index -- a deleted participantId cannot be reused. + +WHAT THIS DOES NOT DO: it does not register your provider. The three rows here +are the adapters' own identities, which they need before they can sign anything +or verify each other. Your upstream API is a Participant of type "upstream" +plus a ProviderSchema row, and its base URL is your ngrok tunnel -- so you +create those two by hand. README.md has the curl. + +Needs python3 and the cryptography package: + + pip install cryptography +""" +import base64, json, os, pathlib, sys, time, urllib.error, urllib.parse, urllib.request + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey +from cryptography.hazmat.primitives import serialization as ser + +ROOT = pathlib.Path(__file__).resolve().parent.parent +KEYS = ROOT / "keys" / "keys.json" +ADAPTERS = ROOT / "config" / "adapters" + + +def load_dotenv(): + """Read .env into the environment. + + There is no Makefile here to source it first, and a real environment + variable wins so a one-off override still works: + + REGISTRY_PORT=9081 python3 bin/setup.py + """ + path = ROOT / ".env" + if not path.exists(): + sys.exit("setup: no .env -- copy .env.example to .env first") + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, value = line.split("=", 1) + os.environ.setdefault(name.strip(), value.strip()) + + +def env(name, default=None): + v = os.environ.get(name, default) + if v is None: + sys.exit(f"setup: {name} is not set -- is it missing from .env?") + return v + + +# --------------------------------------------------------------------- keys + +def b64(raw): + return base64.b64encode(raw).decode() + + +def ed25519_pair(): + k = Ed25519PrivateKey.generate() + return (b64(k.private_bytes(ser.Encoding.Raw, ser.PrivateFormat.Raw, ser.NoEncryption())), + b64(k.public_key().public_bytes(ser.Encoding.Raw, ser.PublicFormat.Raw))) + + +def x25519_pair(): + k = X25519PrivateKey.generate() + return (b64(k.private_bytes(ser.Encoding.Raw, ser.PrivateFormat.Raw, ser.NoEncryption())), + b64(k.public_key().public_bytes(ser.Encoding.Raw, ser.PublicFormat.Raw))) + + +def load_or_generate_keys(): + """Keys persist across runs: the registry already holds the public halves.""" + roles = (("exp", env("EXP_SUBSCRIBER_ID")), + ("network", env("NETWORK_SUBSCRIBER_ID")), + ("provider", env("PROVIDER_SUBSCRIBER_ID"))) + + if KEYS.exists(): + print("keys: reusing keys/keys.json") + identities = json.loads(KEYS.read_text()) + # The keypair persists; the id is read from .env each run. This registry + # cannot update a record, so changing an id in .env seeds a NEW + # participant rather than editing one -- and the same keypair moving to + # the new id is what makes that a rename rather than a rekey. + for role, participant in roles: + if identities[role]["participantId"] != participant: + print(f" {role}: id is now {participant}, keeping the keypair") + identities[role]["participantId"] = participant + KEYS.write_text(json.dumps(identities, indent=2)) + return identities + + print("keys: generating") + identities = {} + for role, participant in roles: + sign_private, sign_public = ed25519_pair() + encr_private, encr_public = x25519_pair() + identities[role] = {"participantId": participant, + "signingPrivate": sign_private, "signingPublic": sign_public, + "encrPrivate": encr_private, "encrPublic": encr_public} + KEYS.parent.mkdir(exist_ok=True) + KEYS.write_text(json.dumps(identities, indent=2)) + KEYS.chmod(0o600) + return identities + + +# ----------------------------------------------------------------- registry + +def registry_url(): + return f"http://localhost:{env('REGISTRY_PORT', '8081')}" + + +def token(): + body = urllib.parse.urlencode({ + "client_id": env("KEYCLOAK_CLIENT_ID", "registry-frontend"), + "grant_type": "password", + "username": env("REGISTRY_USER", "no-user"), + "password": env("REGISTRY_PASSWORD", "no-user-password"), + }).encode() + # Keycloak sits behind PROXY_ADDRESS_FORWARDING, so it builds the token's + # issuer from these headers. Without them it answers with an empty body. + # + # keycloak:8080 is the CONTAINER-INTERNAL address, and is deliberately not + # KEYCLOAK_PORT. The registry validates the issuer against + # OAUTH2_RESOURCES_0_URI, which names that internal address -- so a token + # minted with the host port in its issuer is rejected with a 401 and an + # empty body, however the port is published. + req = urllib.request.Request( + f"http://localhost:{env('KEYCLOAK_PORT', '8080')}/auth/realms/" + f"{env('KEYCLOAK_REALM', 'sunbird-rc')}/protocol/openid-connect/token", + data=body, headers={"X-Forwarded-Host": "keycloak:8080", + "X-Forwarded-Proto": "http"}) + with urllib.request.urlopen(req, timeout=30) as r: + payload = json.load(r) + if not payload.get("access_token"): + sys.exit("setup: keycloak issued no token -- check the KEYCLOAK_* values in .env") + return payload["access_token"] + + +def post(entity, payload, bearer): + # No {"EntityName": {...}} wrapper: this registry takes the record itself, + # and a wrapper comes back as "extraneous key [...] is not permitted". + req = urllib.request.Request(f"{registry_url()}/api/v1/{entity}", method="POST", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {bearer}"}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + except urllib.error.HTTPError as e: + # The registry answers a rejected write with a JSON envelope, but not + # always: a 401 comes back with an empty body. Decoding blindly turns + # that into a JSONDecodeError traceback that says nothing about what + # went wrong, so report the status instead. + raw = e.read() + try: + return json.loads(raw) + except ValueError: + detail = raw.decode(errors="replace").strip()[:200] or "(empty body)" + sys.exit(f"setup: the registry refused a write -- HTTP {e.code}: {detail}\n" + f" A 401 here usually means the token was minted for a different\n" + f" issuer than the registry validates against.") + + +def search(entity, filters): + req = urllib.request.Request(f"{registry_url()}/api/v1/{entity}/search", method="POST", + data=json.dumps({"filters": filters}).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r).get("data", []) + + +def wait_for_registry(): + for _ in range(60): + try: + search("Participant", {}) + return + except Exception: + time.sleep(2) + sys.exit("setup: the registry did not come up -- check `docker compose ps`\n" + " and that REGISTRY_PORT in .env matches the published port") + + +def signing_key_block(public_key): + # The base64: label is the registry's own encoding marker; the adapter + # strips it before the value reaches signature validation. + return [{"keyId": "k1", "use": "sign", "alg": "ed25519", + "key": f"base64:{public_key}", "status": "active", + "validFrom": "2026-01-01T00:00:00Z", "validUntil": "2030-01-01T00:00:00Z"}] + + +def node(participant_id, name, role, public_key): + """A participant that speaks Beckn. + + One level, no wrapper object: type decides which fields apply. baseUrl must + be https for a node, and the id must be hostname-shaped -- it is the + identity that goes on the wire as context.bapId / bppId. Neither is + resolved here: routing between the adapters is the router plugin's config, + which uses the compose service names.""" + return {"participantId": participant_id, "name": name, "type": "node", + "status": "active", "baseUrl": f"https://{participant_id}/beckn", + "role": role, "keys": signing_key_block(public_key)} + + +def ensure_participant(bearer, participant_id, payload): + """Create only when absent. Delete here is soft and keeps the unique index, + so a recreate would fail on a duplicate key rather than replacing.""" + if search("Participant", {"participantId": {"eq": participant_id}}): + print(f" {participant_id}: already present") + return + result = post("Participant", payload, bearer) + status = result["params"]["status"] + print(f" {participant_id}: {status} {result['params'].get('errmsg', '')[:160]}") + + +def seed(identities): + wait_for_registry() + bearer = token() + print("registry: the three adapter identities") + + for role, name, beckn_role in ( + ("exp", "OAN experience layer adapter", "BAP"), + ("network", "OAN network layer adapter", "NETWORK"), + ("provider", "OAN provider layer adapter", "BPP")): + identity = identities[role] + ensure_participant(bearer, identity["participantId"], + node(identity["participantId"], name, beckn_role, + identity["signingPublic"])) + + +def key_osids(identities): + """Read back each key's osid, and check the registry still holds the public + half we have the private half for. + + The Authorization header names a key by its osid rather than by the friendly + id, so the adapters have to be configured with the value the registry + assigned. + + The mismatch check matters because this registry cannot update a record and + its delete is soft: a participant seeded against an earlier keys.json keeps + that public key forever. Signing with a new private half would then produce + signatures nobody can verify -- and the failure would surface much later, as + an authentication error with no obvious cause.""" + for role, identity in identities.items(): + records = search("Participant", {"participantId": {"eq": identity["participantId"]}}) + keys = (records[0].get("keys") or []) if records else [] + if not keys: + sys.exit(f"setup: {identity['participantId']} has no published key") + + published = keys[0]["key"].removeprefix("base64:") + if published != identity["signingPublic"]: + sys.exit( + f"setup: {identity['participantId']} is registered with a different key.\n" + f" This registry cannot update a record, and its delete is soft and keeps\n" + f" the unique index, so the id cannot be reused. Either restore the\n" + f" matching keys/keys.json, or pick a new id for {role.upper()}_SUBSCRIBER_ID\n" + f" in .env and re-run.") + identity["keyOsid"] = keys[0]["osid"] + return identities + + +# ------------------------------------------------------------------ configs + +def render(identities): + print("configs:") + binding = f"{env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')}" + for role in ("exp", "network", "provider"): + identity = identities[role] + template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() + prefix = role.upper() + for placeholder, value in ( + (f"__{prefix}_SUBSCRIBER_ID__", identity["participantId"]), + (f"__{prefix}_KEY_ID__", identity["keyOsid"]), + (f"__{prefix}_SIGNING_PRIVATE__", identity["signingPrivate"]), + (f"__{prefix}_SIGNING_PUBLIC__", identity["signingPublic"]), + (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), + (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"]), + ("__PROVIDER_BINDING_KEY__", binding)): + template = template.replace(placeholder, value) + if "__" in template: + sys.exit(f"setup: {role}.yaml still has unrendered placeholders") + out = ADAPTERS / f"{role}.yaml" + out.write_text(template) + out.chmod(0o600) # holds a private key + print(f" config/adapters/{role}.yaml") + + +if __name__ == "__main__": + load_dotenv() + identities = load_or_generate_keys() + seed(identities) + identities = key_osids(identities) + KEYS.write_text(json.dumps(identities, indent=2)) + render(identities) + print(f""" +ready -- the adapters can now sign and verify each other. + +Still to do, by hand, because the base URL is yours: + + 1. start your upstream API locally and expose it + ngrok http 9100 + 2. register it -- two rows, see README.md: + Participant type "upstream", baseUrl = your https ngrok URL + ProviderSchema bindingKey {env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')} + 3. docker compose up -d + +The bindingKey above is what the provider adapter was just configured to answer +to. If the row you create says anything else, the adapter passes the request +through and you get a bare ACK with no on_select.""") diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/docker-deployment/config/adapters/exp.yaml.tmpl new file mode 100644 index 0000000..1b3c3c9 --- /dev/null +++ b/docker-deployment/config/adapters/exp.yaml.tmpl @@ -0,0 +1,66 @@ +# exp-adapter +# +# The caller. Signs outbound requests as oan-caller and routes them by action: +# discovery to the network layer, transactions straight to the provider adapter. +# No validateSign -- the experience app calling it is inside the trust boundary, +# so there is no network signature to check on the way in. +appName: "exp-adapter" + +log: + level: debug + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + port: 9202 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +modules: + - name: exp-adapter + # A subtree: every action lands here and the payload says which one it is. + path: /beckn/ + handler: + type: std + role: bap + subscriberId: __EXP_SUBSCRIBER_ID__ + + plugins: + registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + + keyManager: + id: simplekeymanager + config: + subscriberId: __EXP_SUBSCRIBER_ID__ + # The KEY's osid, not a friendly name: that is what the registry + # indexes keys by, and what a verifier looks up. + keyId: __EXP_KEY_ID__ + signingPrivateKey: "__EXP_SIGNING_PRIVATE__" + signingPublicKey: "__EXP_SIGNING_PUBLIC__" + encrPrivateKey: "__EXP_ENCR_PRIVATE__" + encrPublicKey: "__EXP_ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + router: + id: router + config: + routingConfig: /app/config/routing-exp.yaml + + steps: + - addRoute + - sign diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl new file mode 100644 index 0000000..a005a7b --- /dev/null +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -0,0 +1,69 @@ +# network-adapter +# +# The network layer. Verifies the caller's signature, then hands discovery to +# the discovery service and re-signs as itself on the way out. +appName: "network-adapter" + +log: + level: debug + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + port: 9201 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +modules: + - name: network-adapter + # A subtree: every action lands here and the payload says which one it is. + path: /beckn/ + handler: + type: std + # bpp because this adapter RECEIVES from a BAP. The role decides which + # context identity validateSign compares the signer against: bap expects + # the sender to be the bppId, bpp expects the bapId -- and the sender here + # is the caller, oan-caller. + role: bpp + subscriberId: __NETWORK_SUBSCRIBER_ID__ + + plugins: + registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + + keyManager: + id: simplekeymanager + config: + subscriberId: __NETWORK_SUBSCRIBER_ID__ + # The KEY's osid, not a friendly name: that is what the registry + # indexes keys by, and what a verifier looks up. + keyId: __NETWORK_KEY_ID__ + signingPrivateKey: "__NETWORK_SIGNING_PRIVATE__" + signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" + encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" + encrPublicKey: "__NETWORK_ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + router: + id: router + config: + routingConfig: /app/config/routing-network.yaml + + steps: + - validateSign + - addRoute + - sign diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl new file mode 100644 index 0000000..8a6b109 --- /dev/null +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -0,0 +1,97 @@ +# OAN provider adapter -- local end-to-end run. +# +# Serves /beckn/ synchronously: verifies the sender against the registry, +# resolves the capability's call plan, calls the provider, and answers with the +# mapped result. No callback -- the answer is the HTTP response. +appName: "oan-provider-adapter" + +log: + level: debug + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + +http: + port: 9200 + timeout: + read: 30 + write: 30 + idle: 30 + +pluginManager: + root: ./plugins + +modules: + - name: oanProvider + # A subtree, not one action. Without the trailing slash Go's ServeMux + # matches exactly, so /beckn/select would mount that action and 404 the + # rest. Which action it is comes from the payload, never the URL. + path: /beckn/ + handler: + type: std + role: bpp + subscriberId: __PROVIDER_SUBSCRIBER_ID__ + + plugins: + # Serves both halves: the sender's signing key for validateSign, and the + # capability call plans the provider steps resolve against. + registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + + # The adapter's OWN keys, used by signAck to sign what it answers with. + # Local test keys -- a real deployment uses a key manager backed by a + # secret store, not values in a config file. + keyManager: + id: simplekeymanager + config: + subscriberId: __PROVIDER_SUBSCRIBER_ID__ + # The KEY's osid, not a friendly name: that is what the registry + # indexes keys by, and what a verifier looks up. + keyId: __PROVIDER_KEY_ID__ + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + + signValidator: + id: signvalidator + signer: + id: signer + + # Generic: fetches, compiles and caches whatever the registry's mapping + # URLs point at. Knows nothing about any provider. + mapper: + id: jsonmapper + config: + fetchTimeout: 10s + cacheTTL: 1m + + # One entry per provider capability. Each recognises its own binding key + # and passes through anything else. + # + # authScheme none matches what the registry publishes for this upstream, + # which is what lets its baseUrl be plaintext http. Against a real + # provider this is basic or header, and the credential is named here as + # an environment variable -- never held in this file or in the registry: + # + # authScheme: basic + # usernameEnv: MAUSAMGRAM_USER + # passwordEnv: MAUSAMGRAM_X_API_KEY + # A list: one provider can serve several capabilities, and the registry + # contract expects exactly that -- one Participant, one ProviderSchema + # row per capability. Comma-separated, because a plugin config value is + # a string; a binding key uses a pipe, so a comma is unambiguous. + providerSteps: + - id: weather + config: + bindingKeys: "__PROVIDER_BINDING_KEY__" + authScheme: none + + steps: + - validateSign # the sender's key, from the registry + - weather # resolve, map out, call, map back + - signAck # signs whatever the step answered with diff --git a/docker-deployment/config/adapters/routing-exp.yaml b/docker-deployment/config/adapters/routing-exp.yaml new file mode 100644 index 0000000..d759e9c --- /dev/null +++ b/docker-deployment/config/adapters/routing-exp.yaml @@ -0,0 +1,25 @@ +# Experience layer adapter routing. +# +# Branches by action: discovery goes to the network layer, everything +# transactional goes straight to the provider adapter. For v2.x the router +# ignores domain and matches on version and endpoint alone. +# +# targetType "url" appends the action to the path, so the base here is the +# module mount point and /discover or /select is added to it. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://network-adapter:9201/beckn" + endpoints: + - discover + + - version: "2.0.0" + targetType: "url" + target: + url: "http://provider-adapter:9200/beckn" + endpoints: + - select + - init + - confirm + - status diff --git a/docker-deployment/config/adapters/routing-network.yaml b/docker-deployment/config/adapters/routing-network.yaml new file mode 100644 index 0000000..4419664 --- /dev/null +++ b/docker-deployment/config/adapters/routing-network.yaml @@ -0,0 +1,14 @@ +# Network layer adapter routing. +# +# One job: hand discovery to the discovery service, which is a service in this +# same compose and so reachable by name. +# +# The service serves /discover at its root, so no /beckn prefix here: targetType +# "url" appends the action to whatever base is given. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://discovery:8080" + endpoints: + - discover diff --git a/docker-deployment/config/discovery/instance.yaml.example b/docker-deployment/config/discovery/instance.yaml.example new file mode 100644 index 0000000..1da4c29 --- /dev/null +++ b/docker-deployment/config/discovery/instance.yaml.example @@ -0,0 +1,42 @@ +# Deployment-local overrides — layer three of four, and the only optional one. +# Copy to config/instance.yaml, which is gitignored; a missing instance.yaml is +# not an error. +# +# NO SECRETS. DATABASE_URL and every other credential arrive from the process +# environment, which sits above this file precisely so a secret store beats a +# checked-out path (TRD §8). + +app: + # The network this deployment serves — mahavistar, bharatvistar, and so on. + # On publish it fills an empty publishDirectives.visibleTo (C8). It has no + # repo-wide default because there is no repo-wide answer. + network: mahavistar + +server: + port: 8080 + +database: + # Sized by the concurrency model, not guessed: discover runs its retrieval + # modes concurrently (A2), so one in-flight discover holds as many + # connections as it has enabled modes. + # + # maxConns >= (enabled modes) x (expected in-flight discovers) + # + # Two modes in Phase 1, three once semantic lands. maxConns must also stay + # under the server's own max_connections less whatever else shares it. + # minConns is a warm-start knob only — idle backends cost the server memory + # to save a connection handshake. + maxConns: 32 + minConns: 4 + +log: + level: info + +# Uncomment when an Ollama deployment exists to turn semantic search on (A5). +# embeddings: +# provider: ollama + +# Uncomment to export traces and metrics to a collector (T2). +# otel: +# exporter: otlp +# endpoint: http://localhost:4317 diff --git a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml new file mode 100644 index 0000000..ca8fb12 --- /dev/null +++ b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml @@ -0,0 +1,247 @@ +# Mausamgram, openagrinet:WeatherObservation, select. Both directions, one file. +# +# One file per binding-action rather than one per direction, because both legs of +# one upstream call are one unit of configuration: they are published, reviewed +# and retired together, and a reference to one is a reference to the other. +# +# The registry entry pointing here decides which action this serves, so nothing +# in the file names it. The filename's action segment must match that entry -- +# a mismatch would apply a correct mapping to the wrong call, silently. +# +# Both halves read: +# beckn the inbound Beckn payload -- the context to echo, the offer to +# quote against +# and the response half additionally reads: +# response the provider's answer, in its own shape +# +# Nothing else is in scope. Values the provider step resolved before the call are +# not passed in: the step holds them and used them to make the call, so a mapping +# reading them back would be a second name for the same data. Where the answer +# needs them, it takes them from what the provider echoed. + +# The response half follows the openagrinet:WeatherObservation v0.1 schema pack, +# Direct mode. The pack lives in OpenAgriNet/network-specs; it is referred to +# here by name and version rather than by a path, because a path pins a branch +# and a branch moves. +# +# @context is the canonical schemas.openagrinet.global identifier. In JSON-LD +# that is a name, not a fetch target -- it does not have to resolve today, and +# substituting a raw git URL that does would put an implementation detail on the +# wire and break every consumer when the branch is renamed. +# +# Direct mode requires observationType, source, location, generatedAt and +# parameters. informationMode is what selects those requirements: a catalog +# resource advertising this capability is OnDemand instead, and carries +# supportedParameters rather than values. +# +# The answer returns ONE RESOURCE PER FORECAST DAY, each with its own id derived +# from its date. That is the shape the pack describes: every WeatherObservation +# example carries a single validity and a flat parameters array, so a period is a +# resource and there is no form for several in one. +# +# The ids are therefore new -- the request named an abstract point forecast, the +# answer returns the concrete days that satisfy it. Which is why the offer's +# resourceIds are rewritten below rather than echoed: the offer arrives naming +# the id that was asked for, and leaving it would point the offer at something +# that appears nowhere in the answer. +# +# ONE FIELD HERE IS NOT IN THE PACK, deliberately. The pack sets no +# additionalProperties, so it validates; it is simply not governed. +# +# aggregation The pack's parameter entry is parameter/value/unit only. This +# provider reports a minimum AND a maximum for temperature and +# humidity, which are indistinguishable without it. +# +# Fields that are the same for every day -- the point, the source, the +# observation type -- sit once at the top. Only what varies per day repeats. + +# What this capability requires of a payload, checked before either half runs. +# A predicate that is false refuses the request with the message beside it, so +# the caller is told what is wrong with their payload rather than that an +# expression somewhere returned false. +# +# This rule used to be Go: the step read the geometry and required a Point, which +# meant a capability with a different rule needed a different build. It is here +# now, beside the extraction it guards. +# +# NOTE the consequence: nothing in the adapter enforces a geometry any more. A +# mapping that declares no preconditions accepts whatever arrives and hands it to +# the request half, which is exactly the configurability that was asked for -- +# and exactly why the responsibility sits in this file. +# +# One check, because there is one thing to say. $exists guards the type test, so +# a request carrying no location and a request carrying a Polygon both land here +# and both learn what this capability needs -- splitting them would be two +# entries repeating the same sentence. +# +# Each check is its own expression and binds $ra for itself; there is no shared +# scope with the halves below. Where several checks say genuinely different +# things, they are separate entries and the first failure is the one reported. +required: + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.location) and $ra.location.type = "Point" + ) + message: "this capability needs a Point location; the provider forecasts one point at a time" + +# The request half decides what the provider is asked for. Whatever it produces +# IS the request: query parameters for a method with no body, a body for one that +# takes it. +# +# This is where the extraction lives, deliberately. The step reads only the +# geometry's type -- enough to refuse a Polygon with a clear error, because a +# mapping cannot refuse -- and nothing else. So when this provider wants another +# parameter, it is an edit here and nothing else: no Go, no rebuild, live on the +# next cache expiry. +# +# A date range, for instance, is already in the payload and would be two lines: +# +# "from": $ra.validity.startsAt, +# "to": $ra.validity.endsAt +# +# $ra is bound once so the rest reads as plain field access rather than four +# repetitions of the same path. +# +# GeoJSON is [lon, lat] -- longitude first. Reading them the other way round +# gives a point in the wrong hemisphere that is still a valid request, so it +# fails as wrong data rather than as an error. +request: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + { + "lat": $ra.location.coordinates[1], + "lon": $ra.location.coordinates[0] + } + ) + +# Keyed by direction, not by the action it produces: a select is answered by an +# on_select over the same HTTP round trip, so the callback is this half rather +# than an action of its own. +response: | + ( + $lat := response.location.lat; + $lon := response.location.lon; + /* However many days the provider sent. It answers fcstday1..fcstdayN and N + is whatever the forecast ran to, so naming five would truncate a ten-day + answer and mis-handle a one-day one. + + Sorted on the numeric suffix, not the key: the keys sort lexically as + fcstday1, fcstday10, fcstday2, and a ten-day forecast delivered in that + order would be wrong in a way nothing downstream could detect. */ + $days := $each(response, function($v, $k) { + $contains($k, "fcstday") ? { + "n": $number($substringAfter($k, "fcstday")), + "day": $v + } + })^(n).day; + + $reading := function($name, $aggregation, $unit, $value) { + $exists($value) ? { + "parameter": $name, + "aggregation": $aggregation, + "unit": $unit, + "value": $value + } + }; + + /* A warning is a parameter, not a field of its own: the pack has no + advisory property but does have an Alert parameter. Unit "1" is what it + prescribes for a value that has no unit. */ + $alert := function($value) { + $exists($value) ? { + "parameter": "Alert", + "unit": "1", + "value": $value + } + }; + + $selected := beckn.message.contract.commitments[0]; + + /* Bound once because it is used twice -- for a resource's own id and for the + offer's reference to it. Two copies of the same expression is how a + dangling reference gets reintroduced. */ + $resourceId := function($day) { "res:mausamgram:forecast:" & $day.date }; + + { + "context": { + "version": beckn.context.version, + "action": "on_select", + "networkId": beckn.context.networkId, + "bapId": beckn.context.bapId, + "bapUri": beckn.context.bapUri, + "bppId": beckn.context.bppId, + "bppUri": beckn.context.bppUri, + "transactionId": beckn.context.transactionId, + "messageId": beckn.context.messageId, + "timestamp": $now() + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { "code": "QUOTED", "name": "Quoted" } + }, + /* The offer is echoed, but its references are not: the request + named the abstract point forecast, and the answer returns the + concrete days. Leaving resourceIds as they arrived would point + the offer at an id that appears nowhere in the answer. + + $merge keeps everything else the request offered -- the id, the + descriptor, the provider -- and replaces one key. */ + "offer": $merge([ + $selected.offer, + { "resourceIds": [$map($days, function($day) { $resourceId($day) })] } + ]), + /* One resource per forecast day, which is what the pack describes: + every WeatherObservation example carries a single validity and a + flat parameters array, so a period is a resource and there is no + form for several in one. + + Wrapped for the same reason as the resourceIds above: JSONata + collapses a one-element sequence to a bare value, so a one-day + forecast would answer with an object where every other N answers + with a list. */ + "resources": [$map($days, function($day) { + { + "id": $resourceId($day), + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "informationMode": "Direct", + "observationType": "Forecast", + "subjectCategories": ["Weather"], + "source": { + "sourceId": "mausamgram", + "sourceName": "IMD Mausamgram NWP" + }, + "location": { + "type": "Point", + "coordinates": [$lon, $lat] + }, + "generatedAt": $now(), + /* This resource reports one day, so its validity opens and + closes on it. */ + "validity": { + "startsAt": $day.date, + "endsAt": $day.date + }, + "parameters": [ + $reading("Rainfall", "Total", "mm", $day.rain), + $reading("Temperature", "Minimum", "Cel", $day.tmin), + $reading("Temperature", "Maximum", "Cel", $day.tmax), + $reading("Humidity", "Minimum", "%", $day.rhmin), + $reading("Humidity", "Maximum", "%", $day.rhmax), + $reading("WindSpeed", "Average", "m/s", $day.wspd), + $alert($day.weather_warning ? $day.weather_warning : $day.cloud_message) + ] + } + } + })] + } + ] + } + } + } + ) diff --git a/docker-deployment/config/registry/imports/realm-export.json b/docker-deployment/config/registry/imports/realm-export.json new file mode 100644 index 0000000..0f5efa1 --- /dev/null +++ b/docker-deployment/config/registry/imports/realm-export.json @@ -0,0 +1,2321 @@ +{ + "id": "sunbird-rc", + "realm": "sunbird-rc", + "displayName": "Sunbird Rc Core", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "a772a1cd-7904-4e5c-a864-5041fa69d491", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "42dba8cf-f483-4668-a087-cba46ed86ad2", + "name": "admin", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "5fa4077d-1686-4506-97a6-5bce1bce59dc", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "network_operator", + "description": "Network Operator - onboards and governs Providers", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "registryOperator", + "description": "Registry Operator - manages SchemaRegistry, Participant and ProviderSchema records", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "270ecc82-3249-475c-a851-d3ea162059b8", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "0de39ec0-7602-4aa2-b54d-ab12e9bdb76f", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4259031b-736e-49eb-9e70-4a312a48e211", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9887e071-49b0-464b-b6fe-a1c585a709c7", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7d61f967-1dce-482f-96e5-9eff79eb4851", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-events", + "manage-identity-providers", + "manage-clients", + "view-identity-providers", + "manage-authorization", + "view-users", + "manage-users", + "manage-events", + "manage-realm", + "impersonation", + "view-authorization", + "query-clients", + "create-client", + "view-clients", + "query-users", + "query-realms", + "view-realm", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "e93b1761-fb32-46c5-bfa2-4b853c7b5573", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "bc32a137-07a1-40f0-b9fd-a6e64e27f99b", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4b5abd90-d6a2-4981-a50f-520292496f0b", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "99d2ed5f-00a9-44ed-8b9f-bdd7ba3facb8", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "3fbd2cd5-0698-490e-a52f-ef528d001a62", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9b8b4f1c-5ed6-49ca-bec3-0a9a4867ad26", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7e6341ff-a1d8-4400-af94-3a007a06706a", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ad0c87da-9f34-4992-a83a-f6b924f1944d", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "badb0d4d-06da-45e8-a777-ef47f712d3ed", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "f8f48f0f-bd2a-4cb7-9b77-af69b9805c25", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ca0b1e94-6578-4295-abf4-ae99f7df7595", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ff7230eb-7dae-44a5-8f68-f68747f35589", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "890d054b-86f9-49f5-8dd9-14f62aa956de", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "627f3f4c-58e3-49f3-9989-a05d4d0a8752", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-api": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "2e9bfeff-129e-4072-9617-5847644aac24", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "attributes": {} + } + ], + "account": [ + { + "id": "5694c2d0-6d02-4182-bb09-78f4f5f1ec2d", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0612622f-dae7-48f8-8985-fe7e5ab8acc7", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "eeefbd57-94b8-4d7d-bf2f-075c39ccb746", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "9e9165b9-1170-47ab-802a-aecffefb3ab7", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "a8d0a100-e382-49ba-ac42-48dbf815a2de", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "08772792-146d-4676-ba2d-ce56b0104263", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0a2e7893-784e-47ef-ba35-4a26901350c0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + } + ], + "registry-frontend": [] + } + }, + "groups": [], + "defaultRole": { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "sunbird-rc" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpSupportedApplications": [ + "FreeOTP", + "Google Authenticator" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "users": [ + { + "id": "3cc9ac60-b67d-4c57-8005-acd4d236b2dc", + "createdTimestamp": 1634296700339, + "username": "service-account-admin-api", + "enabled": true, + "totp": false, + "emailVerified": false, + "serviceAccountClientId": "admin-api", + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-sunbird-rc", + "admin" + ], + "clientRoles": { + "realm-management": [ + "manage-users", + "manage-realm" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "username": "no-user", + "enabled": true, + "emailVerified": false, + "credentials": [ + { + "type": "password", + "value": "no-user-password", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-sunbird-rc", + "network_operator", + "registryOperator" + ] + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account" + ] + } + ] + }, + "clients": [ + { + "id": "01326f76-7838-47fc-ae62-399a75c5ae38", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "f871d6fc-d997-4ac6-99fe-d797955bc9f0", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "8ab32c51-9aa0-4e28-80bf-0d6b53151354", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "25962708-6d45-47d9-8935-5db159234aac", + "clientId": "admin-api", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "*", + "http://localhost:4200/", + "http://localhost:4200/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "84ae9d6c-424f-47f0-9d4d-f2e98fed7339", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "98406938-b8db-4992-8519-917054f6ed0e", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "id": "90d6b17a-5a06-4546-8091-960301f8147e", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b245b10b-606c-417c-bbc0-8f81a7a992a6", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "961a8a92-1598-48ff-adee-1e5fee0df757", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "69c04ae8-6669-48e7-8234-08986a7f490d", + "clientId": "registry-frontend", + "name": "Registry Frontend", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "login_theme": "sunbird-rc", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b777b14f-b0e8-4da5-a802-092803319cbe", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/sunbird-rc/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/sunbird-rc/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "7160f35d-97d3-4730-9769-4b03b32e5191", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "b4695333-f842-4ef7-874e-99260e77b9cb", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "fc23d4b8-76c5-4e59-9305-10846b8bcefe", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + }, + { + "id": "08f06ba5-3e60-4a0a-aaf9-f70bfc7ae99e", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e28227ee-cb54-4557-8908-01864f80055f", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "364a632f-b66a-4ca4-8bbe-ec2ce1af9df8", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "42cad815-4de0-4b67-abca-7f7aaf55e589", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "e3302def-d387-465c-a420-7ab01570e94a", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "63a3cb24-b124-428e-ac0f-253eb1fe485d", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2eb041ca-970a-45fd-a167-2a497579bc8c", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "2c02e9ce-7d86-4a5b-84b8-cf93114ddf26", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "2d39b55e-46c4-4dec-bd83-f081c708f544", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "e869fffd-f801-492d-a6c7-d6c6143817e5", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "daeb863b-4773-4668-98fb-403e93414eb2", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "a98b9f93-ec39-4f3c-acb7-cd92161e3717", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "c59a379a-3934-4e6f-be20-1803b0786d97", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0a299a91-277c-4f38-95e7-6c520f892b63", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "d51531b3-a8ea-44e2-a48f-69991f9166cc", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "b4b33a89-01db-468e-9a4e-c5ac58304fed", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "d1727e08-fb90-49ce-bb7e-d7a55a50ee64", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "564ae79d-e505-416c-b794-ddd3a3c21fde", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "986d2d9e-0d0d-4317-92b3-a7a8d9bec4de", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "e14ec2e9-0d24-4960-8779-00f769ccc01b", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "8f044609-b615-4522-b9e7-8361cb08b0b3", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "341b838d-ba26-4280-b0af-3e5d3403c938", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "26a17e7e-1a6e-439f-a54a-05a63d1c91fb", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "2489b2c0-5b3a-4404-8428-be4ce653da72", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "8c7e1d96-bf79-42e6-9360-b5e7b8dddc8d", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "01f959d8-123f-4263-ad6e-386e8b4d0e05", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "b47a30d6-3c49-4bbe-b15e-b0eb6cffc0f3", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "2cc8166d-6d77-4a85-9945-bc22b0f550e3", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "1d87a800-cc11-4d85-aa76-8a6d828e2269", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "42cc1538-f83a-4a94-b5a5-d16b80824a02", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "13ff6325-822e-4087-9e74-086de77fe89e", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e501527e-dec8-4fde-a539-8e77d86b5081", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "b4b519d8-070d-4dab-854e-d6e3b2b36205", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + } + ], + "defaultDefaultClientScopes": [ + "role_list", + "profile", + "email", + "roles", + "web-origins" + ], + "defaultOptionalClientScopes": [ + "offline_access", + "address", + "phone", + "microprofile-jwt" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "ed42958b-6e78-42a9-9f40-2e40bd6c8dd0", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-attribute-mapper", + "saml-user-property-mapper", + "oidc-usermodel-property-mapper", + "oidc-address-mapper", + "saml-role-list-mapper", + "oidc-full-name-mapper" + ] + } + }, + { + "id": "572219a7-3053-4940-87c5-ad94a6fb6dd3", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "8704a420-bf90-4e12-9e33-d21f39a2385b", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "8cf98455-916b-487a-8322-3f5d283400c2", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "0b43488d-108b-41f5-ab6d-56a4ac8ff63c", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-role-list-mapper", + "oidc-address-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "6f0ebf9b-900a-4ca9-8fea-90719f218689", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "b5a486b3-abf9-49a1-8dc6-dc5e20776681", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "be43420e-8ffc-4f53-b745-f2f0cd88f000", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "f749bd77-72f2-4dc4-a65e-dd89b255f12f", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "a541cbb1-8a27-4061-a389-9f24ba1c2eb1", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "18504bf7-63f1-4848-b565-6348fa6b0048", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "497d8386-9a74-4b7b-a4e6-78bbbbb5d795", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "d964973c-2106-4db3-a814-f7a34ae7a1ce", + "alias": "Authentication Options", + "description": "Authentication options.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ff1b250-85b1-4709-8719-3eabcb34493f", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "43f683be-52e7-43cd-aa9e-6318b8079ad0", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "5195e46e-b2c3-49e3-8987-db8b19c45fc5", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "aca5480d-842c-4fa9-aff1-b8af8d51d82a", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Account verification options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "66a00ea9-7ec1-4450-905a-14b7f3f8e4bf", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "a24445a4-1988-4b5b-bde6-fa36dbd07e03", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "31c2bc3b-6eb1-4c4f-8464-3528f7445ef7", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4ccc8da9-0e1e-4f30-99c0-e2139f671a80", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "flowAlias": "forms", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "fa1183d1-af7a-40dd-ba85-d7d37867639c", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "f5b5e49b-7cc9-4011-b9fa-60f0ef65e735", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "5cf727d2-e25a-4c88-a55b-4eea9134adb1", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ec4f009-2f16-464b-8feb-a0bdc0dad195", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "User creation or linking", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "a695a5e0-326f-4658-8518-a1769d97ad5f", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4611e0a9-a6a4-4e32-8500-e68877b464b1", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Authentication Options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "55be727b-a17b-40c5-a5c3-c2d72c7f54cb", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "flowAlias": "registration form", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "00ec3b72-3abb-4db3-ad2f-595bc2f7e086", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "82e4f3a3-744b-4d8a-8785-6eabaf9e05c9", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "0c0312b2-db7c-433c-ab15-20b18bfb5f4a", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "ee0faa63-999c-42e6-8189-c22a5cc14dc5", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "506eed8f-88c9-4978-b13a-886f1efc45c0", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "oauth2DevicePollingInterval": "5", + "clientOfflineSessionMaxLifespan": "0", + "clientSessionIdleTimeout": "0", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5" + }, + "keycloakVersion": "14.0.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} diff --git a/docker-deployment/config/registry/schemas/Participant.json b/docker-deployment/config/registry/schemas/Participant.json new file mode 100644 index 0000000..e1e8a09 --- /dev/null +++ b/docker-deployment/config/registry/schemas/Participant.json @@ -0,0 +1,536 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Participant", + "type": "object", + "required": [ + "Participant" + ], + "properties": { + "Participant": { + "$ref": "#/definitions/Participant" + } + }, + "definitions": { + "Participant": { + "description": "Someone the network deals with. `type` says which kind, and decides which of the remaining fields apply: a node speaks Beckn and is addressed by its participantId; an upstream is an ordinary API our adapter calls.", + "type": "object", + "additionalProperties": false, + "required": [ + "participantId", + "name", + "type", + "status", + "baseUrl" + ], + "properties": { + "participantId": { + "$ref": "#/definitions/ParticipantId" + }, + "name": { + "description": "Human label.", + "type": "string", + "minLength": 1, + "maxLength": 200, + "pattern": "\\S" + }, + "type": { + "$ref": "#/definitions/ParticipantType" + }, + "status": { + "$ref": "#/definitions/Status" + }, + "baseUrl": { + "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https, except that an upstream with auth.scheme 'none' may be plaintext.", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + }, + "role": { + "description": "What this node does on the network. BAP asks. BPP answers. NETWORK exposes publish and discover, answering discover from published catalogs.", + "type": "string", + "enum": [ + "BAP", + "BPP", + "NETWORK" + ] + }, + "keys": { + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used in the Authorization keyId.", + "type": "array", + "minItems": 1, + "maxItems": 8, + "uniqueItems": true, + "items": { + "$ref": "#/definitions/PublicKey" + } + }, + "auth": { + "$ref": "#/definitions/Auth" + } + }, + "allOf": [ + { + "description": "A node speaks Beckn: it needs a role and keys, has no credential of ours to present, and its id is its wire identity so it must be a hostname.", + "if": { + "properties": { + "type": { + "const": "node" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "role", + "keys" + ], + "not": { + "anyOf": [ + { + "required": [ + "auth" + ] + } + ] + }, + "properties": { + "participantId": { + "maxLength": 253, + "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$" + }, + "baseUrl": { + "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + } + } + } + }, + { + "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify, only a credential we present.", + "if": { + "properties": { + "type": { + "const": "upstream" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "auth" + ], + "not": { + "anyOf": [ + { + "required": [ + "role" + ] + }, + { + "required": [ + "keys" + ] + } + ] + } + } + }, + { + "description": "A credential over plaintext http is a leaked credential.", + "if": { + "properties": { + "auth": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "not": { + "const": "none" + } + } + } + } + }, + "required": [ + "auth" + ] + }, + "then": { + "properties": { + "baseUrl": { + "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + } + } + } + } + ] + }, + "PublicKey": { + "description": "A public key, held as material: it is public, so there is nothing to protect. Contrast Secret, always a pointer.", + "type": "object", + "additionalProperties": false, + "required": [ + "keyId", + "use", + "alg", + "key", + "validFrom", + "status" + ], + "properties": { + "keyId": { + "description": "Second field of the Authorization keyId, so a sender can say which key it signed with.", + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" + }, + "use": { + "description": "sign verifies signatures; encrypt is for encrypted payloads.", + "type": "string", + "enum": [ + "sign", + "encrypt" + ] + }, + "alg": { + "description": "Fixed by use. Both curves are 32-byte keys.", + "type": "string", + "enum": [ + "ed25519", + "x25519" + ] + }, + "key": { + "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. A truncated or wrong-curve key fails at write time.", + "type": "string", + "pattern": "^base64:[A-Za-z0-9+/]{43}=$" + }, + "validFrom": { + "type": "string", + "format": "date-time" + }, + "validUntil": { + "description": "Absent means open-ended. Overlap with the next key's validFrom is the rotation window.", + "type": "string", + "format": "date-time" + }, + "status": { + "description": "revoked withdraws the key early. Effective only once the reader refreshes.", + "type": "string", + "enum": [ + "active", + "revoked" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "use": { + "const": "sign" + } + } + }, + "then": { + "properties": { + "alg": { + "const": "ed25519" + } + } + } + }, + { + "if": { + "properties": { + "use": { + "const": "encrypt" + } + } + }, + "then": { + "properties": { + "alg": { + "const": "x25519" + } + } + } + } + ] + }, + "Auth": { + "description": "How our adapter authenticates TO an upstream. Not Beckn signing — that uses Node.keys.", + "type": "object", + "additionalProperties": false, + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "type": "string", + "enum": [ + "none", + "apiKeyQuery", + "apiKeyHeader", + "basic" + ] + }, + "paramName": { + "description": "The single query-parameter or header name the credential goes in.", + "$ref": "#/definitions/ParamName" + }, + "valuePrefix": { + "description": "Prepended to the credential, trailing space included — 'Bearer '. Header schemes only.", + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,30} $" + }, + "paramNames": { + "description": "For an upstream wanting several named credentials. Keys must match `secrets` exactly.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/definitions/ParamName" + } + }, + "secrets": { + "description": "Pointers to credentials held outside the registry. Redacted from /search by _osConfig.privateFields.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/definitions/Secret" + } + } + }, + "allOf": [ + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "const": "none" + } + } + }, + "then": { + "allOf": [ + { + "not": { + "required": [ + "secrets" + ] + } + }, + { + "not": { + "required": [ + "paramName" + ] + } + }, + { + "not": { + "required": [ + "paramNames" + ] + } + }, + { + "not": { + "required": [ + "valuePrefix" + ] + } + } + ] + } + }, + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "enum": [ + "apiKeyQuery", + "apiKeyHeader" + ] + } + } + }, + "then": { + "required": [ + "secrets" + ], + "oneOf": [ + { + "required": [ + "paramName" + ], + "not": { + "required": [ + "paramNames" + ] + }, + "properties": { + "secrets": { + "maxProperties": 1 + } + } + }, + { + "required": [ + "paramNames" + ], + "allOf": [ + { + "not": { + "required": [ + "paramName" + ] + } + }, + { + "not": { + "required": [ + "valuePrefix" + ] + } + } + ] + } + ] + } + }, + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "const": "apiKeyQuery" + } + } + }, + "then": { + "not": { + "required": [ + "valuePrefix" + ] + } + } + }, + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "const": "basic" + } + } + }, + "then": { + "required": [ + "secrets" + ], + "properties": { + "secrets": { + "required": [ + "username", + "password" + ] + } + }, + "allOf": [ + { + "not": { + "required": [ + "paramName" + ] + } + }, + { + "not": { + "required": [ + "paramNames" + ] + } + }, + { + "not": { + "required": [ + "valuePrefix" + ] + } + } + ] + } + } + ] + }, + "ParamName": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" + }, + "ParticipantId": { + "description": "Stable id, and the only id. For a node this is its network identity (context.bapId / context.bppId); for an upstream it is the Beckn offer.provider.id.", + "type": "string", + "maxLength": 253, + "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" + }, + "Status": { + "type": "string", + "enum": [ + "active", + "inactive" + ] + }, + "MaterialRef": { + "description": "A pointer to SECRET material held outside the registry, :. Never the material. Public keys are not MaterialRefs: PublicKey.key holds the bytes.", + "type": "string", + "maxLength": 1024, + "pattern": "^(env://[A-Z][A-Z0-9_]{0,63}|inline:[!-~][ -~]{0,998})$" + }, + "Secret": { + "$ref": "#/definitions/MaterialRef" + }, + "ParticipantType": { + "description": "node speaks Beckn. upstream is an API we call over ordinary HTTP and does not.", + "type": "string", + "enum": [ + "node", + "upstream" + ] + } + }, + "_osConfig": { + "uniqueIndexFields": [ + "participantId" + ], + "indexFields": [ + "status", + "type", + "baseUrl" + ], + "privateFields": [ + "$.auth.secrets" + ], + "roles": [ + "registryOperator" + ], + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ] + } +} diff --git a/docker-deployment/config/registry/schemas/ProviderSchema.json b/docker-deployment/config/registry/schemas/ProviderSchema.json new file mode 100644 index 0000000..16f5200 --- /dev/null +++ b/docker-deployment/config/registry/schemas/ProviderSchema.json @@ -0,0 +1,83 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ProviderSchema", + "type": "object", + "required": ["ProviderSchema"], + "properties": { "ProviderSchema": { "$ref": "#/definitions/ProviderSchema" } }, + + "definitions": { + + "ProviderSchema": { + "description": "One row is one provider and one capability. What varies per Beckn action — the URL, the method, the mapping, the timeout — varies inside actions[].", + "type": "object", + "additionalProperties": false, + "required": ["bindingKey", "participantId", "capabilityCode", "status", "actions"], + "properties": { + "bindingKey": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{2,63}\\|openagrinet:[A-Z][A-Za-z0-9]*$", + "not": { "pattern": "\\|openagrinet:Agriculture(Capability|Resource)$" } + }, + "participantId": { "$ref": "#/definitions/ParticipantId" }, + "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, + "status": { "$ref": "#/definitions/Status" }, + + "actions": { + "description": "At least one. uniqueItems compares whole objects, so it cannot pin one entry per action — verify/records.py does.", + "type": "array", + "minItems": 1, + "maxItems": 10, + "items": { "$ref": "#/definitions/ActionBinding" } + } + } + }, + + "ActionBinding": { + "description": "How to call this provider for one Beckn action: where, how, and with which mapping. Anything the upstream needs that the Beckn body cannot express is the adapter plugin's work, not a field here. status is per action, so one can be retired without touching the others.", + "type": "object", + "additionalProperties": false, + "required": ["action", "method", "path", "mappings", "status"], + "properties": { + "action": { "$ref": "#/definitions/Action" }, + "method": { "type": "string", "enum": ["GET", "POST"] }, + "path": { "$ref": "#/definitions/Path" }, + "mappings": { "$ref": "#/definitions/MappingPath" }, + + "timeoutMs": { "type": "integer", "minimum": 1000, "maximum": 120000, "default": 15000 }, + "retryMax": { "type": "integer", "minimum": 0, "maximum": 5, "default": 0 }, + + "status": { "$ref": "#/definitions/Status" } + } + }, + + "Action": { + "description": "beckn v2.0.0 request actions. An on_* callback is not one: it is the response half of the action that made the call.", + "type": "string", + "enum": ["discover", "select", "init", "confirm", "status", + "track", "cancel", "update", "rate", "support"] + }, + + "MappingPath": { + "description": "Fully-qualified URL of one published mapping file, holding request: and response: as YAML block scalars. The action segment must equal the action it sits under — verify/records.py.", + "type": "string", + "maxLength": 2000, + "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*/[A-Za-z0-9._~%-]*\\.(discover|select|init|confirm|status|track|cancel|update|rate|support)\\.ya?ml$" + }, + + "ParticipantId": { "type": "string", "maxLength": 253, "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" }, + "CapabilityCode": { "type": "string", + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$", + "not": { "pattern": "^openagrinet:Agriculture(Capability|Resource)$" } }, + "Status": { "type": "string", "enum": ["active", "inactive"] }, + "Path": { "type": "string", "maxLength": 512, + "description": "Appended to that upstream's baseUrl. Single slashes: an empty segment is never deliberate, and many servers answer //a differently from /a. A trailing slash is allowed, because /api/ and /api are a distinction some APIs make.", + "pattern": "^(?!.*//)/[A-Za-z0-9._~%/-]*$" } + }, + + "_osConfig": { + "uniqueIndexFields": ["bindingKey"], + "indexFields": ["participantId", "capabilityCode", "status"], + "roles": ["registryOperator"], + "systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"] + } +} diff --git a/docker-deployment/config/registry/schemas/SchemaRegistry.json b/docker-deployment/config/registry/schemas/SchemaRegistry.json new file mode 100644 index 0000000..f67b7fe --- /dev/null +++ b/docker-deployment/config/registry/schemas/SchemaRegistry.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SchemaRegistry", + "type": "object", + "required": ["SchemaRegistry"], + "properties": { "SchemaRegistry": { "$ref": "#/definitions/SchemaRegistry" } }, + + "definitions": { + + "SchemaRegistry": { + "type": "object", + "additionalProperties": false, + "required": ["capabilityCode", "name", "version", "schemaUrl", "status"], + "properties": { + "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, + "name": { "type": "string", "minLength": 1, "maxLength": 200, "pattern": "\\S" }, + "version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+$" }, + "schemaUrl": { "type": "string", "maxLength": 2000, + "pattern": "^https://raw\\.githubusercontent\\.com/OpenAgriNet/network-specs/[A-Za-z0-9_-]+/schema/[A-Za-z0-9]+/v[0-9]+\\.[0-9]+/[A-Za-z0-9_-]+\\.yaml$" }, + "status": { "$ref": "#/definitions/Status" } + } + }, + + "CapabilityCode": { "type": "string", + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$", + "not": { "pattern": "^openagrinet:Agriculture(Capability|Resource)$" } }, + "Status": { "type": "string", "enum": ["active", "inactive"] } + }, + + "_osConfig": { + "uniqueIndexFields": ["capabilityCode"], + "indexFields": ["status"], + "roles": ["registryOperator"], + "systemFields": ["osCreatedAt", "osUpdatedAt", "osCreatedBy", "osUpdatedBy"] + } +} diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml new file mode 100644 index 0000000..405bca6 --- /dev/null +++ b/docker-deployment/docker-compose.yml @@ -0,0 +1,233 @@ +# The whole OAN stack in one file: registry, discovery, and the three adapters. +# +# 1. cp .env.example .env and read it -- ADAPTER_REF matters +# 2. docker compose up -d registry and discovery first come up +# 3. python3 bin/setup.py keys, the three adapter entries, configs +# 4. docker compose up -d again, so the adapters read the rendered configs +# +# There is deliberately NO provider here. You run your own upstream API +# locally, expose it with ngrok, and register it yourself -- see README.md. +# Nothing in this file knows the provider exists; the adapter learns its base +# URL from the registry at request time. + +x-adapter: &adapter + image: ${ADAPTER_IMAGE:-oan/network-adapter:local} + build: + # A remote git context, so this stack needs no sibling checkout. Point + # ADAPTER_SRC at a local path instead when you are working on the adapter. + context: ${ADAPTER_SRC} + dockerfile: Dockerfile.adapter-with-plugins + restart: unless-stopped + environment: &adapter-env + CONFIG_FILE: /app/config/adapter.yaml + +services: + # ---------------------------------------------------------------- registry + registry-db: + image: postgres:14 + container_name: oan-registry-db + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - registry-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] + interval: 5s + timeout: 5s + retries: 20 + + keycloak: + image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest + container_name: oan-keycloak + restart: unless-stopped + volumes: + - ./config/registry/imports:/opt/jboss/keycloak/imports + environment: + - DB_VENDOR=postgres + - DB_ADDR=registry-db + - DB_PORT=5432 + - DB_DATABASE=${POSTGRES_DB} + - DB_USER=${POSTGRES_USER} + - DB_PASSWORD=${POSTGRES_PASSWORD} + - KEYCLOAK_USER=${KEYCLOAK_ADMIN_USER} + - KEYCLOAK_PASSWORD=${KEYCLOAK_ADMIN_PASSWORD} + - KEYCLOAK_IMPORT=/opt/jboss/keycloak/imports/realm-export.json + - PROXY_ADDRESS_FORWARDING=true + ports: + - "${KEYCLOAK_PORT}:8080" + - "${KEYCLOAK_ADMIN_PORT}:9990" + depends_on: + registry-db: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9990/ || exit 1"] + interval: 10s + timeout: 10s + retries: 30 + + registry: + image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_VERSION} + container_name: oan-registry + restart: unless-stopped + volumes: + # Schemas are read at startup, so a change here needs this service + # restarted before the registry will honour it. + - ./config/registry/schemas:/home/sunbirdrc/config/public/_schemas + environment: + - connectionInfo_uri=jdbc:postgresql://registry-db:5432/${POSTGRES_DB} + - connectionInfo_username=${POSTGRES_USER} + - connectionInfo_password=${POSTGRES_PASSWORD} + - search_providerName=dev.sunbirdrc.registry.service.NativeSearchService + - authentication_enabled=true + - sunbird_sso_realm=${KEYCLOAK_REALM} + - sunbird_sso_url=http://keycloak:8080/auth + # Without this the service resolves its issuer to localhost -- which + # inside the container is itself -- and the security filter chain fails + # to build, so the whole registry never starts. + - OAUTH2_RESOURCES_0_URI=http://keycloak:8080/auth/realms/${KEYCLOAK_REALM} + - OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH=realm_access.roles + - identity_provider=dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl + - sunbird_sso_admin_client_id=${KEYCLOAK_ADMIN_CLIENT_ID} + - sunbird_sso_client_id=${KEYCLOAK_CLIENT_ID} + - sunbird_sso_admin_client_secret=${KEYCLOAK_SECRET} + - sunbird_keycloak_user_set_password=true + - sunbird_keycloak_user_password=${REGISTRY_DEFAULT_USER_PASSWORD} + - encryption_enabled=false + - event_enabled=false + - idgen_enabled=false + - claims_enabled=false + - did_enabled=false + - signature_enabled=false + - certificate_enabled=false + - filestorage_enabled=false + - notification_enabled=false + - notification_async_enabled=false + - async_enabled=false + - webhook_enabled=false + - registry_base_apis_enable=false + - manager_type=DefinitionsManager + - expand_reference=false + - swagger_title=OAN Registry + - logging.level.root=INFO + ports: + - "${REGISTRY_PORT}:8081" + depends_on: + registry-db: + condition: service_healthy + keycloak: + condition: service_healthy + healthcheck: + # This image ships no curl, wget, nc or bash, so the check reads the + # kernel's own table instead: 1F91 is 8081 in hex, and the service binds + # v6, hence tcp6. It proves the port is accepting connections, which is + # what everything downstream waits for. + test: ["CMD-SHELL", "grep -q ':1F91' /proc/net/tcp6"] + interval: 5s + timeout: 5s + retries: 60 + start_period: 40s + + # --------------------------------------------------------------- discovery + discovery-db: + # pgvector rather than plain postgres: the discovery service's HNSW index + # options arrived in 0.8. + image: pgvector/pgvector:0.8.0-pg16 + container_name: oan-discovery-db + restart: unless-stopped + environment: + POSTGRES_USER: discovery + POSTGRES_PASSWORD: discovery + POSTGRES_DB: discovery + volumes: + - discovery-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U discovery -d discovery"] + interval: 5s + timeout: 5s + retries: 20 + + discovery: + image: ${DISCOVERY_IMAGE:-oan/discovery-service:local} + build: + context: ${DISCOVERY_SRC} + container_name: oan-discovery + restart: unless-stopped + environment: + DATABASE_URL: postgres://discovery:discovery@discovery-db:5432/discovery?sslmode=disable + DATABASE_AUTO_MIGRATE: "true" + APP_NETWORK_ID: ${APP_NETWORK_ID} + SERVER_PORT: 8080 + VALIDATION_SPEC_URL: ${BECKN_SPEC_URL} + # config/common.yaml is baked into the image and is the reviewed default. + # To override a setting, copy config/discovery/instance.yaml.example to + # config/discovery/instance.yaml and uncomment the mount below -- compose + # creates a DIRECTORY if you mount a file that does not exist, which the + # service then fails to parse, so the file has to be there first. + # volumes: + # - ./config/discovery/instance.yaml:/app/config/instance.yaml:ro + ports: + - "${DISCOVERY_PORT}:8080" + depends_on: + discovery-db: + condition: service_healthy + + # ---------------------------------------------------------------- adapters + # Verifies the caller, calls your upstream provider, answers synchronously. + # It has no provider address of its own: it reads the ProviderSchema row you + # register, so pointing this at a new ngrok URL is a registry edit, not a + # config change here. + provider-adapter: + <<: *adapter + container_name: oan-provider-adapter + depends_on: + registry: + condition: service_healthy + environment: + <<: *adapter-env + volumes: + - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro + ports: + - "${PROVIDER_ADAPTER_PORT}:9200" + + # Verifies the caller, hands discovery on, re-signs as itself. + network-adapter: + <<: *adapter + container_name: oan-network-adapter + depends_on: + registry: + condition: service_healthy + discovery: + condition: service_started + environment: + <<: *adapter-env + volumes: + - ./config/adapters/network.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro + ports: + - "${NETWORK_ADAPTER_PORT}:9201" + + # The caller, and the only one that takes unsigned requests: the experience + # app is inside the trust boundary, so there is no network signature to + # check. This is what makes the stack testable with a plain curl. + exp-adapter: + <<: *adapter + container_name: oan-exp-adapter + depends_on: + network-adapter: + condition: service_started + provider-adapter: + condition: service_started + environment: + <<: *adapter-env + volumes: + - ./config/adapters/exp.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/routing-exp.yaml:/app/config/routing-exp.yaml:ro + ports: + - "${EXP_ADAPTER_PORT}:9202" + +volumes: + registry-data: + discovery-data: From 2e6be5ef887149d99cf63ae1eae02876a04ee02a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:46:12 +0530 Subject: [PATCH 02/81] refactor: target the compose deployment at a dev VM [OpenAgriNet/network-adapter#4] The folder read as a laptop sandbox. It is a shared dev environment on a VM, and the difference matters in three places. Ports now bind to 127.0.0.1 rather than every interface, overridable with BIND_ADDR. Behind them sit a Keycloak whose admin password ships as "admin" and a registry whose write token any reader of .env.example can mint, so the old default published the whole network's identity records, writable, to whatever the VM was reachable from. The README documents the ssh -L tunnel and says what has to be true before BIND_ADDR is widened. .env.example no longer claims nothing in it is secret. Every credential in it is a shipped default, which is the same as public, and it says so. The three adapter identities are now *.oan.dev rather than *.oan.local, and the discovery network id oan-dev rather than local-network. The registry is append-only and an id can never be reused, so a misleading name is permanent. Also drops the four identity fields from the response context in the shipped mapping, matching the adapter's own copy: bapId, bapUri, bppId and bppUri. A mapping transforms a payload and has no business asserting network identity, and the two Uri fields it copied were whatever the caller sent -- which in a deployed stack is a container-internal address that means nothing outside it. Identity on an answer is the signature the adapter puts on it. --- docker-deployment/.env.example | 84 +++++++--- docker-deployment/README.md | 156 ++++++++++++------ docker-deployment/bin/setup.py | 23 +-- .../config/adapters/provider.yaml.tmpl | 7 +- .../weather-observation.select.yaml | 16 +- docker-deployment/docker-compose.yml | 45 +++-- 6 files changed, 216 insertions(+), 115 deletions(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index b698380..6326e14 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -1,17 +1,32 @@ -# Copy to .env and read it through once. Nothing here is a real secret: every -# value is local to this stack, and the adapter keypairs are generated by -# bin/setup.py into keys/keys.json rather than being written here. +# Copy to .env and read it through once. +# +# This is a DEV deployment on a VM, not a laptop sandbox. Every credential +# below is a shipped default, which means it is public -- change all of them +# before the stack is reachable by anyone but you. The adapter keypairs are the +# exception: bin/setup.py generates those into keys/keys.json, and they are +# never written here. + +# ---- who can reach it ------------------------------------------------------ +# The interface the published ports bind to. 127.0.0.1 keeps the stack on the +# VM's loopback, reachable over an SSH tunnel: +# +# ssh -L 9202:127.0.0.1:9202 -L 8081:127.0.0.1:8081 you@the-vm +# +# Set this to 0.0.0.0 only once something in front of it is terminating TLS +# and authenticating. Behind these ports sit Keycloak's admin console and a +# registry whose write token any reader of this file can mint. +BIND_ADDR=127.0.0.1 # ---- where the images come from -------------------------------------------- -# A remote git context, so no sibling checkout is needed: docker clones it. +# A remote git context, so no checkout is needed on the VM: docker clones it. # The fragment after # is the ref. # # It points at the feature branch because the three OAN plugins -- oanregistry, # jsonmapper and the weather provider step -- are not on the default branch # yet. Change it to #development once that has merged. # -# Working on the adapter? Point this at a local path instead: -# ADAPTER_SRC=../../network-adapter +# Working on the adapter? Point this at a path on the VM instead: +# ADAPTER_SRC=/srv/network-adapter ADAPTER_SRC=https://github.com/OpenAgriNet/network-adapter.git#feat/41-oan-adapter-plugins DISCOVERY_SRC=https://github.com/OpenAgriNet/discovery-service.git @@ -20,7 +35,7 @@ DISCOVERY_SRC=https://github.com/OpenAgriNet/discovery-service.git ADAPTER_IMAGE=oan/network-adapter:local DISCOVERY_IMAGE=oan/discovery-service:local -# ---- ports (host side) ----------------------------------------------------- +# ---- ports ----------------------------------------------------------------- REGISTRY_PORT=8081 KEYCLOAK_PORT=8080 KEYCLOAK_ADMIN_PORT=9990 @@ -30,6 +45,7 @@ NETWORK_ADAPTER_PORT=9201 EXP_ADAPTER_PORT=9202 # ---- registry -------------------------------------------------------------- +# CHANGE every credential in this block before the stack is exposed. REGISTRY_VERSION=v2.0.0 POSTGRES_DB=registry POSTGRES_USER=postgres @@ -45,41 +61,57 @@ REGISTRY_USER=no-user REGISTRY_PASSWORD=no-user-password # ---- discovery ------------------------------------------------------------- -APP_NETWORK_ID=local-network +APP_NETWORK_ID=oan-dev BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml # ---- the three adapter identities ------------------------------------------ # bin/setup.py registers exactly these three in the registry and generates a -# keypair for each. A participant id IS the network identity -- what goes on -# the wire as context.bapId / bppId -- so the registry requires it to be -# hostname-shaped. These names are never resolved by DNS: routing between the -# adapters is the router plugin's config, which uses the compose service names. -EXP_SUBSCRIBER_ID=exp.oan.local -NETWORK_SUBSCRIBER_ID=network.oan.local -PROVIDER_SUBSCRIBER_ID=provider.oan.local +# keypair for each. +# +# A participant id IS the network identity -- what goes on the wire as +# context.bapId / bppId -- so the registry requires it to be hostname-shaped. +# They are never resolved by DNS: routing between the adapters is the router +# plugin's config, which uses the compose service names. +# +# Pick these once and deliberately. The registry is append-only: there is no +# update, delete is soft, and a soft-deleted id keeps the unique index -- so an +# id can never be reused. Name them for the environment they are, so a dev +# identity cannot be mistaken for a real one on a shared network. +EXP_SUBSCRIBER_ID=exp.oan.dev +NETWORK_SUBSCRIBER_ID=network.oan.dev +PROVIDER_SUBSCRIBER_ID=provider.oan.dev -# ---- YOUR provider --------------------------------------------------------- -# You create these two registry rows by hand -- see README.md. setup.py does -# NOT create them, because the base URL is your own ngrok tunnel. +# ---- the upstream provider ------------------------------------------------- +# The provider's two registry rows are created by hand -- see README.md. +# setup.py does not create them, because the base URL belongs to whoever runs +# the upstream API. +# +# WHY THESE TWO MUST MATCH THOSE ROWS. The provider adapter decides whether a +# request is its own by building a binding key out of the incoming payload -- +# the provider id and the capability @type it carries -- and comparing that +# against the keys in its own config. setup.py renders these two values into +# that config as the one key it answers to. # -# These two values must MATCH the rows you create: they are rendered into the -# provider adapter's config as the binding key it answers to. A mismatch means -# the adapter passes your request straight through and you get a bare ACK with -# no on_select, which looks like nothing happened. +# When they disagree the step does not fail. It concludes the request is meant +# for some other provider and passes it through untouched, which is exactly +# what lets one adapter host several capabilities. Nothing further answers, so +# the reply is a bare {"status":"ACK"} with no on_select -- the request looks +# accepted and silently does nothing. These must therefore equal participantId +# and capabilityCode in the ProviderSchema row, exactly. PROVIDER_PARTICIPANT_ID=my-weather-api PROVIDER_CAPABILITY=openagrinet:WeatherObservation # The mapping the provider adapter fetches, request and response in one file. # The registry row holds the full URL and the adapter fetches it verbatim. # -# This defaults to the published copy, which is the same file you will find in +# This defaults to the published copy, which is the same file found in # config/mappings/ -- that copy is there to read and to fork, not to be served # from here. A mapping has to be published somewhere the adapter can reach # before it can be tested, so what this stack exercises is what consumers # actually fetch. Serving the local copy would prove the file works and prove # nothing about the file anyone else reads. # -# Forking it? Publish your copy (a raw GitHub URL is fine) and put that URL in -# the ProviderSchema row you create. This variable is only the default the -# README's example curl uses. +# To change it: fork it, publish the copy (a raw GitHub URL is fine) and put +# that URL in the ProviderSchema row. This variable is only the default the +# README example uses. MAPPING_URL=https://raw.githubusercontent.com/ameersohel45/oan-mappings/main/mausamgram/weather-observation.select.yaml diff --git a/docker-deployment/README.md b/docker-deployment/README.md index e3d9de2..df0fd6b 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -1,36 +1,61 @@ # OAN stack, on Docker Compose -The whole OpenAgriNet stack on one machine: the registry, the discovery -service, and the three adapters. One compose file, one config folder. +The whole OpenAgriNet stack for a **dev deployment on a VM**: the registry, the +discovery service, and the three adapters. One compose file, one config folder. -This is for trying the network end to end on your own laptop. It is not a -production deployment — the keys live in a config file and nothing runs TLS. +This is a dev environment. It is not production: the adapter signing keys sit +in a config file on disk, nothing terminates TLS, and every credential shipped +in `.env.example` is a public default. ## What is here, and what is not Running here: -- **registry** — SunbirdRC, plus its Postgres and Keycloak. Holds who is on - the network, their public keys, and which upstream API answers which - capability. +- **registry** — SunbirdRC, plus its Postgres and Keycloak. Holds who is on the + network, their public keys, and which upstream API answers which capability. - **discovery** — catalogue search, plus its own Postgres. - **three adapters** — experience, network and provider. Same image, three configs. Deliberately **not** here: -- **your provider API.** You run it yourself and expose it with ngrok. The - provider adapter never has its address in a config file — it reads it from - the registry at request time, so swapping tunnels is a registry edit. -- **the provider's registry rows.** You create those two by hand, because the - base URL is your tunnel. `bin/setup.py` registers only the three adapters. +- **the provider API.** Whoever is testing runs it themselves and gives it a + URL the VM can reach. The provider adapter never has that address in a config + file — it reads it from the registry per request, so repointing it is a + registry write and nothing more. +- **the provider's registry rows.** Those two are created by hand, because the + base URL belongs to whoever runs the API. `bin/setup.py` registers only the + three adapters. + +## Reaching it + +Ports bind to `127.0.0.1` on the VM by default. That is deliberate. Behind them +are a Keycloak whose admin password ships as `admin`, and a registry whose +write token anyone who has read `.env.example` can mint. On a VM's public +interface that is the whole network's identity records, writable. + +So reach it over a tunnel: + +```sh +ssh -L 9202:127.0.0.1:9202 -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 you@the-vm +``` + +Then everything below works against `127.0.0.1` on your own machine. + +Set `BIND_ADDR=0.0.0.0` in `.env` only once something in front is terminating +TLS and authenticating, and only after the credentials in `.env` have been +changed. ## Before you start +On the VM: + - Docker with Compose v2 - Python 3 and the `cryptography` package — `pip install cryptography` -- [ngrok](https://ngrok.com/) or any other way to give your local API a public - https URL + +And a URL the VM can reach for the upstream provider API. If that API runs on +someone's laptop, [ngrok](https://ngrok.com/) or any equivalent tunnel gives it +one. ## Bring it up @@ -38,17 +63,19 @@ Deliberately **not** here: cp .env.example .env ``` -Read `.env` before going on. Two things in it matter: +Read `.env` before going on. Four things in it matter: +- **the credentials.** All shipped defaults. Change them. +- `BIND_ADDR` — see above. - `ADAPTER_SRC` points at a **feature branch**, because the three OAN plugins are not on the adapter's default branch yet. -- `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` have to match the - registry rows you create further down. +- `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` have to match the registry + rows created further down. Then: ```sh -# 1. registry and discovery. The adapters will fail to start for now -- +# 1. registry and discovery. The adapters will restart in a loop for now -- # their configs do not exist yet, which step 2 fixes. docker compose up -d @@ -67,49 +94,51 @@ Check it: ```sh docker compose ps -curl -s -X POST http://localhost:8081/api/v1/Participant/search \ +curl -s -X POST http://127.0.0.1:8081/api/v1/Participant/search \ -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool ``` Three participants, one per adapter. That is what `setup.py` seeded. -## Register your provider +## Register the provider Two rows. Both by hand, and both need a token. -Start your API and expose it: +Get the upstream API's URL first. If it is tunnelled from a laptop: ```sh ngrok http 9100 ``` -Take the `https://` URL ngrok prints. Get a token: +Take the `https://` URL. Then get a token: ```sh TOKEN=$(curl -s -X POST \ - "http://localhost:8080/auth/realms/sunbird-rc/protocol/openid-connect/token" \ + "http://127.0.0.1:8080/auth/realms/sunbird-rc/protocol/openid-connect/token" \ -H 'X-Forwarded-Host: keycloak:8080' -H 'X-Forwarded-Proto: http' \ -d 'client_id=registry-frontend' -d 'grant_type=password' \ -d 'username=no-user' -d 'password=no-user-password' \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') ``` -Those two `X-Forwarded-*` headers are not optional. Keycloak runs behind -`PROXY_ADDRESS_FORWARDING` here, and without them it answers with an empty -body. +Those two `X-Forwarded-*` headers are not optional, and `keycloak:8080` is the +**container-internal** address on purpose — not whatever `KEYCLOAK_PORT` is +published as. Keycloak builds the token's issuer from these headers, and the +registry validates that issuer against the internal address. Get it wrong and +the registry rejects the token with a 401 and an empty body. **Row one — the API itself.** Type `upstream`: it has no role and no keys, because it has never heard of Beckn. ```sh -curl -s -X POST http://localhost:8081/api/v1/Participant \ +curl -s -X POST http://127.0.0.1:8081/api/v1/Participant \ -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ -d '{ "participantId": "my-weather-api", "name": "My weather API", "type": "upstream", "status": "active", - "baseUrl": "https://YOUR-NGROK-SUBDOMAIN.ngrok-free.app", + "baseUrl": "https://YOUR-TUNNEL-SUBDOMAIN.ngrok-free.app", "auth": { "scheme": "none" } }' ``` @@ -117,7 +146,7 @@ curl -s -X POST http://localhost:8081/api/v1/Participant \ **Row two — which capability it answers, and how to call it.** ```sh -curl -s -X POST http://localhost:8081/api/v1/ProviderSchema \ +curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema \ -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ -d '{ "bindingKey": "my-weather-api|openagrinet:WeatherObservation", @@ -143,8 +172,7 @@ Things worth knowing about these two calls: permitted`. - **`bindingKey` is `participantId|capabilityCode`.** It has to match what `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` were set to in `.env` - when you ran `setup.py`, because that is the key the provider adapter was - configured to answer to. + when `setup.py` last ran — see the troubleshooting note on bare ACKs. - **`path` must start with one `/` and contain no empty segment.** The schema refuses `//get-daily`, and so does the adapter. - **This registry is append-only.** There is no update, delete is soft, and a @@ -153,18 +181,18 @@ Things worth knowing about these two calls: ## Test it end to end -Replace `my-weather-api` if you used a different id, and point the coordinates -at wherever your API has data. +Replace `my-weather-api` if a different id was used, and point the coordinates +at wherever the API has data. ```sh -curl -s -X POST http://localhost:9202/beckn/select \ +curl -s -X POST http://127.0.0.1:9202/beckn/select \ -H 'Content-Type: application/json' \ -d '{ "context": { "version": "2.0.0", "action": "select", - "networkId": "local-network", - "bapId": "exp.oan.local", "bapUri": "http://exp-adapter:9202/beckn", - "bppId": "provider.oan.local", "bppUri": "http://provider-adapter:9200/beckn", + "networkId": "oan-dev", + "bapId": "exp.oan.dev", "bapUri": "http://exp-adapter:9202/beckn", + "bppId": "provider.oan.dev", "bppUri": "http://provider-adapter:9200/beckn", "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", "timestamp": "2026-09-02T06:12:01.330Z" @@ -192,6 +220,15 @@ curl -s -X POST http://localhost:9202/beckn/select \ You should get an `on_select` back, with one resource per forecast day. +Two things about identity here: + +- The **request** carries `bapId` / `bppId`, because that is the caller saying + who it is. The adapter needs it to look the sender's key up in the registry. +- The **answer** does not echo them back. A mapping transforms a payload; it + does not assert who anyone is, and the `*Uri` fields it could copy are + container-internal addresses that mean nothing outside this compose network. + Identity on the answer is the adapter's signature over it. + The experience adapter is the only one that takes an unsigned request — the experience app is inside the trust boundary, so there is no network signature to check. That is what makes this testable with a plain curl. @@ -231,21 +268,29 @@ tested. What this stack exercises is therefore what consumers actually fetch. Serving the local copy would prove the file works and prove nothing about the file anyone else reads. -To change the mapping: fork it, publish your copy anywhere that serves raw -text over https, and put that URL in the `mappings` field of your -ProviderSchema row. +To change the mapping: fork it, publish the copy anywhere that serves raw text +over https, and put that URL in the `mappings` field of the ProviderSchema row. -The adapter caches a mapping for `cacheTTL` (one minute, in the adapter -config) and GitHub's raw CDN caches for about five, so give an edit a few -minutes to show up. +The adapter caches a mapping for `cacheTTL` (one minute, in the adapter config) +and GitHub's raw CDN caches for about five, so give an edit a few minutes to +show up. ## When it does not work -**`{"status":"ACK"}` and no `on_select`.** The provider adapter did not -recognise the request as its own, so it passed it through. The binding key in -your `ProviderSchema` row does not match what the adapter is configured for. -Compare the row against `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` in -`.env`, and re-run `bin/setup.py` if you change them. +**`{"status":"ACK"}` and no `on_select`.** The commonest one. The provider +adapter did not recognise the request as its own, so it passed it through. + +It decides that by building a binding key from the incoming payload — the +provider id at `message.contract.commitments[].offer.provider.id` and the +capability at `...resources[].resourceAttributes.@type` — and comparing it +against the keys in its own config, which `setup.py` rendered from +`PROVIDER_PARTICIPANT_ID|PROVIDER_CAPABILITY`. + +A mismatch is not an error, by design: passing through is what lets one adapter +host several capabilities. But nothing further answers, so the request looks +accepted and silently does nothing. Compare all three — the payload, the +`ProviderSchema` row, and `.env` — and re-run `bin/setup.py` after changing +`.env`. **The adapters restart in a loop on the first `up`.** Expected before `bin/setup.py` has run — there is no `config/adapters/*.yaml` yet. @@ -254,13 +299,17 @@ Compare the row against `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` in The registry waits on Keycloak, which waits on Postgres, so a cold start takes a minute or two. -**`setup.py` says a participant is registered with a different key.** You have -a `keys/keys.json` that no longer matches the registry. Restore the old one, or +**`setup.py` says a participant is registered with a different key.** There is a +`keys/keys.json` that no longer matches the registry. Restore the old one, or pick new `*_SUBSCRIBER_ID` values in `.env` — the old ids cannot be reused. +**The registry refuses a write with HTTP 401 and an empty body.** The token was +minted for a different issuer than the registry validates against. Check the +`X-Forwarded-Host` header is `keycloak:8080` and not the published port. + **A build fails on `go mod download`, or the adapter cannot fetch the mapping, -with "network is unreachable".** Your machine advertises IPv6 but cannot route -it. Add this to the adapter service in the compose file: +with "network is unreachable".** The host advertises IPv6 but cannot route it. +Add this to the adapter service in the compose file: ```yaml sysctls: @@ -277,4 +326,5 @@ rm -rf keys config/adapters/exp.yaml config/adapters/network.yaml config/adapter ``` Then start again from `docker compose up -d`. New keys mean new identities, so -the provider rows have to be created again too. +the provider rows have to be created again too — and the old participant ids +cannot be reused. diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index e1ecf07..23d1a47 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -9,11 +9,11 @@ alone rather than recreated, because this registry's delete is soft and holds the unique index -- a deleted participantId cannot be reused. -WHAT THIS DOES NOT DO: it does not register your provider. The three rows here +WHAT THIS DOES NOT DO: it does not register the provider. The three rows here are the adapters' own identities, which they need before they can sign anything -or verify each other. Your upstream API is a Participant of type "upstream" -plus a ProviderSchema row, and its base URL is your ngrok tunnel -- so you -create those two by hand. README.md has the curl. +or verify each other. The upstream API is a Participant of type "upstream" plus +a ProviderSchema row, and its base URL belongs to whoever runs it -- so those +two are created by hand. README.md has the curl. Needs python3 and the cryptography package: @@ -299,15 +299,16 @@ def render(identities): print(f""" ready -- the adapters can now sign and verify each other. -Still to do, by hand, because the base URL is yours: +Still to do by hand, because the base URL is not this stack's to know: - 1. start your upstream API locally and expose it - ngrok http 9100 + 1. give the upstream API a URL this VM can reach. Tunnelled from a laptop, + that is: ngrok http 9100 2. register it -- two rows, see README.md: - Participant type "upstream", baseUrl = your https ngrok URL + Participant type "upstream", baseUrl = that https URL ProviderSchema bindingKey {env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')} 3. docker compose up -d -The bindingKey above is what the provider adapter was just configured to answer -to. If the row you create says anything else, the adapter passes the request -through and you get a bare ACK with no on_select.""") +The bindingKey above is what the provider adapter was just configured to +answer to. If the ProviderSchema row says anything else, the adapter concludes +the request is not its own and passes it through -- and the reply is a bare +ACK with no on_select, which looks like nothing happened.""") diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 8a6b109..cb34bcf 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -1,4 +1,4 @@ -# OAN provider adapter -- local end-to-end run. +# OAN provider adapter -- dev deployment. # # Serves /beckn/ synchronously: verifies the sender against the registry, # resolves the capability's call plan, calls the provider, and answers with the @@ -43,8 +43,9 @@ modules: providerEntity: ProviderSchema # The adapter's OWN keys, used by signAck to sign what it answers with. - # Local test keys -- a real deployment uses a key manager backed by a - # secret store, not values in a config file. + # Dev keys, rendered here by bin/setup.py. Production uses a key + # manager backed by a secret store, not values in a config file -- + # which is why this file is gitignored and written 0600. keyManager: id: simplekeymanager config: diff --git a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml index ca8fb12..e63b014 100644 --- a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml +++ b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml @@ -164,14 +164,22 @@ response: | $resourceId := function($day) { "res:mausamgram:forecast:" & $day.date }; { + /* Correlation only: the ids that tie this answer to the request that + asked for it, and nothing that asserts who anybody is. + + bapId, bapUri, bppId and bppUri are deliberately absent. A mapping is + a payload transformation -- it has no business asserting network + identity, and the two Uri fields it could copy are whatever the caller + happened to send, which in a deployed stack is a container-internal + address that means nothing to anyone outside it. Echoing them would + republish another party's routing details as if they were ours. + + Identity on the wire is the adapter's own: it signs what it answers + with, using the key the registry publishes for it. */ "context": { "version": beckn.context.version, "action": "on_select", "networkId": beckn.context.networkId, - "bapId": beckn.context.bapId, - "bapUri": beckn.context.bapUri, - "bppId": beckn.context.bppId, - "bppUri": beckn.context.bppUri, "transactionId": beckn.context.transactionId, "messageId": beckn.context.messageId, "timestamp": $now() diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 405bca6..9855d1f 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -1,14 +1,23 @@ # The whole OAN stack in one file: registry, discovery, and the three adapters. +# Meant for a shared DEV deployment on a VM. # -# 1. cp .env.example .env and read it -- ADAPTER_REF matters -# 2. docker compose up -d registry and discovery first come up +# 1. cp .env.example .env and read it -- every credential in it +# is a default that must be changed +# 2. docker compose up -d registry and discovery come up # 3. python3 bin/setup.py keys, the three adapter entries, configs -# 4. docker compose up -d again, so the adapters read the rendered configs +# 4. docker compose up -d again, so the adapters read the configs # -# There is deliberately NO provider here. You run your own upstream API -# locally, expose it with ngrok, and register it yourself -- see README.md. -# Nothing in this file knows the provider exists; the adapter learns its base -# URL from the registry at request time. +# Ports bind to 127.0.0.1 by default. That is deliberate: this stack has an +# admin-console Keycloak and a registry whose write token any reader of +# .env.example can mint, so publishing it on a VM's public interface would +# hand over the whole network's identity records. Reach it over an SSH tunnel, +# or put a reverse proxy that terminates TLS and authenticates in front, and +# set BIND_ADDR only once something else is doing that job. +# +# There is deliberately NO provider here. The upstream API is run and exposed +# by whoever is testing -- see README.md. Nothing in this file knows the +# provider exists; the adapter learns its base URL from the registry at +# request time, so repointing it is a registry write and nothing more. x-adapter: &adapter image: ${ADAPTER_IMAGE:-oan/network-adapter:local} @@ -57,8 +66,8 @@ services: - KEYCLOAK_IMPORT=/opt/jboss/keycloak/imports/realm-export.json - PROXY_ADDRESS_FORWARDING=true ports: - - "${KEYCLOAK_PORT}:8080" - - "${KEYCLOAK_ADMIN_PORT}:9990" + - "${BIND_ADDR:-127.0.0.1}:${KEYCLOAK_PORT}:8080" + - "${BIND_ADDR:-127.0.0.1}:${KEYCLOAK_ADMIN_PORT}:9990" depends_on: registry-db: condition: service_healthy @@ -113,7 +122,7 @@ services: - swagger_title=OAN Registry - logging.level.root=INFO ports: - - "${REGISTRY_PORT}:8081" + - "${BIND_ADDR:-127.0.0.1}:${REGISTRY_PORT}:8081" depends_on: registry-db: condition: service_healthy @@ -169,16 +178,16 @@ services: # volumes: # - ./config/discovery/instance.yaml:/app/config/instance.yaml:ro ports: - - "${DISCOVERY_PORT}:8080" + - "${BIND_ADDR:-127.0.0.1}:${DISCOVERY_PORT}:8080" depends_on: discovery-db: condition: service_healthy # ---------------------------------------------------------------- adapters - # Verifies the caller, calls your upstream provider, answers synchronously. - # It has no provider address of its own: it reads the ProviderSchema row you - # register, so pointing this at a new ngrok URL is a registry edit, not a - # config change here. + # Verifies the caller, calls the upstream provider, answers synchronously. + # It has no provider address of its own: it reads the ProviderSchema row, + # so repointing it at a different upstream is a registry write, not a + # change here or a restart. provider-adapter: <<: *adapter container_name: oan-provider-adapter @@ -190,7 +199,7 @@ services: volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro ports: - - "${PROVIDER_ADAPTER_PORT}:9200" + - "${BIND_ADDR:-127.0.0.1}:${PROVIDER_ADAPTER_PORT}:9200" # Verifies the caller, hands discovery on, re-signs as itself. network-adapter: @@ -207,7 +216,7 @@ services: - ./config/adapters/network.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro ports: - - "${NETWORK_ADAPTER_PORT}:9201" + - "${BIND_ADDR:-127.0.0.1}:${NETWORK_ADAPTER_PORT}:9201" # The caller, and the only one that takes unsigned requests: the experience # app is inside the trust boundary, so there is no network signature to @@ -226,7 +235,7 @@ services: - ./config/adapters/exp.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-exp.yaml:/app/config/routing-exp.yaml:ro ports: - - "${EXP_ADAPTER_PORT}:9202" + - "${BIND_ADDR:-127.0.0.1}:${EXP_ADAPTER_PORT}:9202" volumes: registry-data: From 8ec00b484f8539ec32fbff522c17e9f7ac4c4c5d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 12:58:20 +0530 Subject: [PATCH 03/81] feat: simplify the participant contract and pull images only [OpenAgriNet/network-adapter#4] Registry schema, four changes: role consumer, provider and network replace BAP, BPP and NETWORK. A role now says what a party does rather than which Beckn acronym it is. Nothing in the adapter compared the old values, so this costs a re-seed and no code. keyId gone. Nothing could look one up: the registry assigns an osid on write, and that osid is what a sender names in the Authorization header, so the friendly id was decoration that read like an identifier. use gone. alg already carried the purpose -- ed25519 signs, x25519 encrypts -- and the adapter treats a missing use as "may sign". The two conditionals that derived alg from use went with it: with use absent each `if` matched vacuously, so both `then` branches applied and alg had to be two curves at once, which refused every key. Found by seeding against the new schema. key bare base64, no "base64:" label. What a verifier hands to a decoder is now the value as published. Auth is removed entirely, along with the paramName, secret and materialRef definitions it used and the privateFields entry that redacted it. The adapter never read it -- it presents credentials from its own config, naming environment variables -- so it described a mechanism that did not exist. Its plaintext-http guard goes too, having guarded a field nothing consumed; if that rule is wanted it belongs where the credential actually lives. Compose no longer builds anything. ADAPTER_IMAGE and DISCOVERY_IMAGE name tags to pull and have no working default, so an unset value fails on pull naming the tag it tried rather than quietly running something else. Verified against a registry running this schema: the three adapter rows seed with the new roles and bare keys, and role "BAP", a prefixed key, a key carrying keyId or use, and an upstream carrying auth are each refused. --- docker-deployment/.env.example | 24 +- docker-deployment/README.md | 27 +- docker-deployment/bin/setup.py | 26 +- .../config/registry/schemas/Participant.json | 351 +----------------- docker-deployment/docker-compose.yml | 24 +- 5 files changed, 69 insertions(+), 383 deletions(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index 6326e14..80f65d6 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -17,23 +17,17 @@ # registry whose write token any reader of this file can mint. BIND_ADDR=127.0.0.1 -# ---- where the images come from -------------------------------------------- -# A remote git context, so no checkout is needed on the VM: docker clones it. -# The fragment after # is the ref. +# ---- the images ------------------------------------------------------------ +# Nothing is built here. `docker compose up -d` pulls these and starts them. # -# It points at the feature branch because the three OAN plugins -- oanregistry, -# jsonmapper and the weather provider step -- are not on the default branch -# yet. Change it to #development once that has merged. +# Set both to the tags published for this environment. There is deliberately no +# working default: an unset or wrong value fails on pull, naming the tag it +# tried, which is a better failure than silently running something else. # -# Working on the adapter? Point this at a path on the VM instead: -# ADAPTER_SRC=/srv/network-adapter -ADAPTER_SRC=https://github.com/OpenAgriNet/network-adapter.git#feat/41-oan-adapter-plugins -DISCOVERY_SRC=https://github.com/OpenAgriNet/discovery-service.git - -# Tags for the images built from the above. Set these to a published image to -# skip building altogether. -ADAPTER_IMAGE=oan/network-adapter:local -DISCOVERY_IMAGE=oan/discovery-service:local +# If they live in a private registry, log in on the VM first: +# docker login ghcr.io +ADAPTER_IMAGE=REPLACE_ME +DISCOVERY_IMAGE=REPLACE_ME # ---- ports ----------------------------------------------------------------- REGISTRY_PORT=8081 diff --git a/docker-deployment/README.md b/docker-deployment/README.md index df0fd6b..c5882fe 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -50,7 +50,9 @@ changed. On the VM: -- Docker with Compose v2 +- Docker with Compose v2, logged in to wherever the images live if it is + private — `docker login ghcr.io` +- the image tags for the adapter and the discovery service - Python 3 and the `cryptography` package — `pip install cryptography` And a URL the VM can reach for the upstream provider API. If that API runs on @@ -65,18 +67,19 @@ cp .env.example .env Read `.env` before going on. Four things in it matter: +- `ADAPTER_IMAGE` and `DISCOVERY_IMAGE` — the tags to pull. No working default; + set them to the tags published for this environment. - **the credentials.** All shipped defaults. Change them. - `BIND_ADDR` — see above. -- `ADAPTER_SRC` points at a **feature branch**, because the three OAN plugins - are not on the adapter's default branch yet. - `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` have to match the registry rows created further down. Then: ```sh -# 1. registry and discovery. The adapters will restart in a loop for now -- -# their configs do not exist yet, which step 2 fixes. +# 1. pulls the images, then starts registry and discovery. The adapters will +# restart in a loop for now -- their configs do not exist yet, which step +# 2 fixes. docker compose up -d # 2. generate the adapter keypairs, register the three adapter identities, @@ -87,9 +90,6 @@ python3 bin/setup.py docker compose up -d ``` -The first run builds the adapter and discovery images from source, so give it -a few minutes. - Check it: ```sh @@ -138,8 +138,7 @@ curl -s -X POST http://127.0.0.1:8081/api/v1/Participant \ "name": "My weather API", "type": "upstream", "status": "active", - "baseUrl": "https://YOUR-TUNNEL-SUBDOMAIN.ngrok-free.app", - "auth": { "scheme": "none" } + "baseUrl": "https://YOUR-TUNNEL-SUBDOMAIN.ngrok-free.app" }' ``` @@ -170,6 +169,14 @@ Things worth knowing about these two calls: - **No `{"Participant": {...}}` wrapper.** The registry takes the record itself. A wrapper comes back as `extraneous key [Participant] is not permitted`. +- **An `upstream` carries no `role`, no `keys` and no credential.** It has + never heard of Beckn, and nothing held in the registry is ever sent to it — + the adapter presents credentials from its own config, naming environment + variables. The schema refuses `role` or `keys` on an upstream. +- **The three roles are `consumer`, `provider` and `network`**, and they apply + to `node` rows only — the three `setup.py` creates. A node also needs at + least one key, published as bare base64 with no encoding label in front of + it. - **`bindingKey` is `participantId|capabilityCode`.** It has to match what `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` were set to in `.env` when `setup.py` last ran — see the troubleshooting note on bare ACKs. diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 23d1a47..76deb85 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -186,10 +186,16 @@ def wait_for_registry(): def signing_key_block(public_key): - # The base64: label is the registry's own encoding marker; the adapter - # strips it before the value reaches signature validation. - return [{"keyId": "k1", "use": "sign", "alg": "ed25519", - "key": f"base64:{public_key}", "status": "active", + """The published half of the signing keypair. + + Bare base64, with no encoding label in front of it: what a verifier hands + to a base64 decoder is the value as published, and a label left on fails + every verification with a decode error pointing nowhere near the registry. + + No friendly key id, because nothing could look one up: the registry assigns + an osid on write and that is what a sender names in the Authorization + header. No use either -- alg carries the purpose, ed25519 signs.""" + return [{"alg": "ed25519", "key": public_key, "status": "active", "validFrom": "2026-01-01T00:00:00Z", "validUntil": "2030-01-01T00:00:00Z"}] @@ -222,13 +228,13 @@ def seed(identities): bearer = token() print("registry: the three adapter identities") - for role, name, beckn_role in ( - ("exp", "OAN experience layer adapter", "BAP"), - ("network", "OAN network layer adapter", "NETWORK"), - ("provider", "OAN provider layer adapter", "BPP")): + for role, name, network_role in ( + ("exp", "OAN experience layer adapter", "consumer"), + ("network", "OAN network layer adapter", "network"), + ("provider", "OAN provider layer adapter", "provider")): identity = identities[role] ensure_participant(bearer, identity["participantId"], - node(identity["participantId"], name, beckn_role, + node(identity["participantId"], name, network_role, identity["signingPublic"])) @@ -251,6 +257,8 @@ def key_osids(identities): if not keys: sys.exit(f"setup: {identity['participantId']} has no published key") + # Bare base64 now, but an older row may still carry the label, and a + # confusing mismatch error is worse than one tolerant line. published = keys[0]["key"].removeprefix("base64:") if published != identity["signingPublic"]: sys.exit( diff --git a/docker-deployment/config/registry/schemas/Participant.json b/docker-deployment/config/registry/schemas/Participant.json index e1e8a09..6d651bc 100644 --- a/docker-deployment/config/registry/schemas/Participant.json +++ b/docker-deployment/config/registry/schemas/Participant.json @@ -40,22 +40,22 @@ "$ref": "#/definitions/Status" }, "baseUrl": { - "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https, except that an upstream with auth.scheme 'none' may be plaintext.", + "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https for a node, since that is its wire identity; an upstream may be plaintext, because nothing in the registry is sent to it.", "type": "string", "maxLength": 2000, "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" }, "role": { - "description": "What this node does on the network. BAP asks. BPP answers. NETWORK exposes publish and discover, answering discover from published catalogs.", + "description": "What this party does on the network. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs.", "type": "string", "enum": [ - "BAP", - "BPP", - "NETWORK" + "consumer", + "provider", + "network" ] }, "keys": { - "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used in the Authorization keyId.", + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header.", "type": "array", "minItems": 1, "maxItems": 8, @@ -63,14 +63,11 @@ "items": { "$ref": "#/definitions/PublicKey" } - }, - "auth": { - "$ref": "#/definitions/Auth" } }, "allOf": [ { - "description": "A node speaks Beckn: it needs a role and keys, has no credential of ours to present, and its id is its wire identity so it must be a hostname.", + "description": "A node speaks Beckn: it needs a role and keys, and its id is its wire identity so it must be a hostname.", "if": { "properties": { "type": { @@ -86,15 +83,6 @@ "role", "keys" ], - "not": { - "anyOf": [ - { - "required": [ - "auth" - ] - } - ] - }, "properties": { "participantId": { "maxLength": 253, @@ -107,7 +95,7 @@ } }, { - "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify, only a credential we present.", + "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify.", "if": { "properties": { "type": { @@ -119,9 +107,6 @@ ] }, "then": { - "required": [ - "auth" - ], "not": { "anyOf": [ { @@ -137,66 +122,22 @@ ] } } - }, - { - "description": "A credential over plaintext http is a leaked credential.", - "if": { - "properties": { - "auth": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "not": { - "const": "none" - } - } - } - } - }, - "required": [ - "auth" - ] - }, - "then": { - "properties": { - "baseUrl": { - "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" - } - } - } } ] }, "PublicKey": { - "description": "A public key, held as material: it is public, so there is nothing to protect. Contrast Secret, always a pointer.", + "description": "A public key, held as material: it is public, so there is nothing to protect. Identified by the osid the registry assigns on write -- which is what a sender names in the Authorization header, and so the only id a verifier can look up. alg carries the purpose: ed25519 signs, x25519 encrypts.", "type": "object", "additionalProperties": false, "required": [ - "keyId", - "use", "alg", "key", "validFrom", "status" ], "properties": { - "keyId": { - "description": "Second field of the Authorization keyId, so a sender can say which key it signed with.", - "type": "string", - "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" - }, - "use": { - "description": "sign verifies signatures; encrypt is for encrypted payloads.", - "type": "string", - "enum": [ - "sign", - "encrypt" - ] - }, "alg": { - "description": "Fixed by use. Both curves are 32-byte keys.", + "description": "The purpose, and the only thing that carries it now: ed25519 signs, x25519 encrypts. Both curves are 32-byte keys.", "type": "string", "enum": [ "ed25519", @@ -204,9 +145,9 @@ ] }, "key": { - "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. A truncated or wrong-curve key fails at write time.", + "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. Bare -- no encoding label in front of it. A truncated or wrong-curve key fails at write time.", "type": "string", - "pattern": "^base64:[A-Za-z0-9+/]{43}=$" + "pattern": "^[A-Za-z0-9+/]{43}=$" }, "validFrom": { "type": "string", @@ -225,260 +166,7 @@ "revoked" ] } - }, - "allOf": [ - { - "if": { - "properties": { - "use": { - "const": "sign" - } - } - }, - "then": { - "properties": { - "alg": { - "const": "ed25519" - } - } - } - }, - { - "if": { - "properties": { - "use": { - "const": "encrypt" - } - } - }, - "then": { - "properties": { - "alg": { - "const": "x25519" - } - } - } - } - ] - }, - "Auth": { - "description": "How our adapter authenticates TO an upstream. Not Beckn signing — that uses Node.keys.", - "type": "object", - "additionalProperties": false, - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "type": "string", - "enum": [ - "none", - "apiKeyQuery", - "apiKeyHeader", - "basic" - ] - }, - "paramName": { - "description": "The single query-parameter or header name the credential goes in.", - "$ref": "#/definitions/ParamName" - }, - "valuePrefix": { - "description": "Prepended to the credential, trailing space included — 'Bearer '. Header schemes only.", - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,30} $" - }, - "paramNames": { - "description": "For an upstream wanting several named credentials. Keys must match `secrets` exactly.", - "type": "object", - "minProperties": 1, - "additionalProperties": { - "$ref": "#/definitions/ParamName" - } - }, - "secrets": { - "description": "Pointers to credentials held outside the registry. Redacted from /search by _osConfig.privateFields.", - "type": "object", - "minProperties": 1, - "additionalProperties": { - "$ref": "#/definitions/Secret" - } - } - }, - "allOf": [ - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "const": "none" - } - } - }, - "then": { - "allOf": [ - { - "not": { - "required": [ - "secrets" - ] - } - }, - { - "not": { - "required": [ - "paramName" - ] - } - }, - { - "not": { - "required": [ - "paramNames" - ] - } - }, - { - "not": { - "required": [ - "valuePrefix" - ] - } - } - ] - } - }, - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "enum": [ - "apiKeyQuery", - "apiKeyHeader" - ] - } - } - }, - "then": { - "required": [ - "secrets" - ], - "oneOf": [ - { - "required": [ - "paramName" - ], - "not": { - "required": [ - "paramNames" - ] - }, - "properties": { - "secrets": { - "maxProperties": 1 - } - } - }, - { - "required": [ - "paramNames" - ], - "allOf": [ - { - "not": { - "required": [ - "paramName" - ] - } - }, - { - "not": { - "required": [ - "valuePrefix" - ] - } - } - ] - } - ] - } - }, - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "const": "apiKeyQuery" - } - } - }, - "then": { - "not": { - "required": [ - "valuePrefix" - ] - } - } - }, - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "const": "basic" - } - } - }, - "then": { - "required": [ - "secrets" - ], - "properties": { - "secrets": { - "required": [ - "username", - "password" - ] - } - }, - "allOf": [ - { - "not": { - "required": [ - "paramName" - ] - } - }, - { - "not": { - "required": [ - "paramNames" - ] - } - }, - { - "not": { - "required": [ - "valuePrefix" - ] - } - } - ] - } - } - ] - }, - "ParamName": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" + } }, "ParticipantId": { "description": "Stable id, and the only id. For a node this is its network identity (context.bapId / context.bppId); for an upstream it is the Beckn offer.provider.id.", @@ -493,15 +181,6 @@ "inactive" ] }, - "MaterialRef": { - "description": "A pointer to SECRET material held outside the registry, :. Never the material. Public keys are not MaterialRefs: PublicKey.key holds the bytes.", - "type": "string", - "maxLength": 1024, - "pattern": "^(env://[A-Z][A-Z0-9_]{0,63}|inline:[!-~][ -~]{0,998})$" - }, - "Secret": { - "$ref": "#/definitions/MaterialRef" - }, "ParticipantType": { "description": "node speaks Beckn. upstream is an API we call over ordinary HTTP and does not.", "type": "string", @@ -520,9 +199,7 @@ "type", "baseUrl" ], - "privateFields": [ - "$.auth.secrets" - ], + "privateFields": [], "roles": [ "registryOperator" ], diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 9855d1f..4c9bdaa 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -1,12 +1,15 @@ # The whole OAN stack in one file: registry, discovery, and the three adapters. # Meant for a shared DEV deployment on a VM. # -# 1. cp .env.example .env and read it -- every credential in it -# is a default that must be changed -# 2. docker compose up -d registry and discovery come up +# 1. cp .env.example .env set the image tags, and change every +# credential -- they are shipped defaults +# 2. docker compose up -d pulls and starts registry and discovery # 3. python3 bin/setup.py keys, the three adapter entries, configs # 4. docker compose up -d again, so the adapters read the configs # +# Nothing is built here. The adapter and discovery images are pulled from the +# tags named in .env. +# # Ports bind to 127.0.0.1 by default. That is deliberate: this stack has an # admin-console Keycloak and a registry whose write token any reader of # .env.example can mint, so publishing it on a VM's public interface would @@ -20,12 +23,10 @@ # request time, so repointing it is a registry write and nothing more. x-adapter: &adapter - image: ${ADAPTER_IMAGE:-oan/network-adapter:local} - build: - # A remote git context, so this stack needs no sibling checkout. Point - # ADAPTER_SRC at a local path instead when you are working on the adapter. - context: ${ADAPTER_SRC} - dockerfile: Dockerfile.adapter-with-plugins + # Pulled, never built. Set ADAPTER_IMAGE in .env to the tag published for + # this environment -- `docker compose up -d` is the whole deployment step. + image: ${ADAPTER_IMAGE} + pull_policy: missing restart: unless-stopped environment: &adapter-env CONFIG_FILE: /app/config/adapter.yaml @@ -159,9 +160,8 @@ services: retries: 20 discovery: - image: ${DISCOVERY_IMAGE:-oan/discovery-service:local} - build: - context: ${DISCOVERY_SRC} + image: ${DISCOVERY_IMAGE} + pull_policy: missing container_name: oan-discovery restart: unless-stopped environment: From 03b8001d5d8dd75db407f26af41d2ba6adc78d77 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 13:31:05 +0530 Subject: [PATCH 04/81] feat: route publish from the provider adapter through the network layer [OpenAgriNet/network-adapter#4] publish now enters at the provider adapter and reaches the discovery service by way of the network layer, which is what fronts discovery for discover already. select is untouched and still answered at the provider adapter without going near it. It needed more than one routing file, and the reason is worth recording. The routing step fails any request whose action is absent from its config, so it cannot be added to the provider adapter's existing /beckn/ module: that module would then have to list select as well, and listing select would proxy it to the network layer instead of answering it locally, which is the only thing that module does. So publishing gets its own module and its own routing config, mounted at /catalog/ and reached as POST /catalog/publish. The catalogue body carries no bapId or bppId, and needs none. The provider adapter signs the forwarded request as itself and the identity travels in the Authorization header's keyId, taken from its keyManager config; the network layer verifies that signature, and its identity check skips a body that declares no caller rather than demanding one. Verified against the local stack: POST /catalog/publish answers ACCEPTED with the catalogue's stats, and select and discover both still work -- select answered locally with its per-day resources, discover returning catalogs. --- docker-deployment/README.md | 40 ++++++++++++- .../config/adapters/provider.yaml.tmpl | 56 +++++++++++++++++++ .../config/adapters/routing-network.yaml | 13 +++-- .../config/adapters/routing-provider.yaml | 20 +++++++ docker-deployment/docker-compose.yml | 1 + 5 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 docker-deployment/config/adapters/routing-provider.yaml diff --git a/docker-deployment/README.md b/docker-deployment/README.md index c5882fe..f070734 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -240,6 +240,41 @@ The experience adapter is the only one that takes an unsigned request — the experience app is inside the trust boundary, so there is no network signature to check. That is what makes this testable with a plain curl. +## How a request flows + +Three paths, and which adapter answers is the whole design: + +``` +discover you -> exp -> network -> discovery service +select you -> exp -> provider -> your upstream API +publish provider (/catalog/publish) -> network -> discovery service +``` + +`discover` and `publish` both end at the discovery service, and both go +through the network adapter — that adapter is what fronts discovery, verifies +the caller and re-signs. `select` never touches it: it goes straight to the +provider adapter, which answers from your upstream API. + +Publishing enters at the **provider** adapter, on its own mount: + +```sh +curl -s -X POST http://127.0.0.1:9200/catalog/publish \ + -H 'Content-Type: application/json' -d @your-catalog.json +``` + +Two things about that: + +- **It is a separate module, `/catalog/`, not another action on `/beckn/`.** + That is forced, not chosen: the routing step fails any request whose action + is missing from its routing config, so a router on `/beckn/` would have to + list `select` as well — and listing it would proxy `select` away instead of + answering it there, which is the one thing that module exists for. +- **The catalogue body needs no `bapId` or `bppId`.** The provider adapter + signs the forwarded request as itself, and the identity travels in the + `Authorization` header's `keyId`, taken from its `keyManager` config. The + network adapter verifies that signature; its identity check skips a body + that declares no caller, so there is nothing to fill in. + ## The layout ``` @@ -251,8 +286,9 @@ config/ exp.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered provider.yaml.tmpl files hold private keys and are gitignored. - routing-exp.yaml which action goes to which adapter - routing-network.yaml + routing-exp.yaml which action goes where. exp splits discover + routing-network.yaml from select; network sends discover and publish + routing-provider.yaml to discovery; provider sends publish onward registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. Read at startup -- a change needs the registry diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index cb34bcf..06db40e 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -96,3 +96,59 @@ modules: - validateSign # the sender's key, from the registry - weather # resolve, map out, call, map back - signAck # signs whatever the step answered with + # The outbound leg: publishing a catalogue to the network layer. + # + # A SECOND module, not another action on /beckn/, and that is forced rather + # than chosen. addRoute fails any request whose action is missing from its + # routing config, so a router on /beckn/ would have to list select too -- + # and listing select would proxy it away instead of answering it here, which + # is the one thing that module exists to do. + # + # POST /catalog/publish. No validateSign: the caller is the provider's own + # catalogue system, inside its trust boundary, exactly as the experience + # adapter takes unsigned calls from the app in front of it. + # + # Nothing here needs bapId or bppId in the body. This adapter signs the + # forwarded request as itself, and the Authorization header's keyId carries + # the identity -- taken from keyManager below, not from the payload. The + # network layer's identity check skips a body that declares no caller. + - name: oanProviderPublish + path: /catalog/ + handler: + type: std + # bap because this module SENDS. The role decides which context identity + # is compared against a signer, and nothing is verified on the way in + # here -- but the same role picks the identity used on the way out. + role: bap + subscriberId: __PROVIDER_SUBSCRIBER_ID__ + + plugins: + registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + + keyManager: + id: simplekeymanager + config: + subscriberId: __PROVIDER_SUBSCRIBER_ID__ + keyId: __PROVIDER_KEY_ID__ + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + + signer: + id: signer + + router: + id: router + config: + routingConfig: /app/config/routing-provider.yaml + + steps: + - addRoute # publish -> the network layer + - sign # as this provider, so the network layer can verify it + diff --git a/docker-deployment/config/adapters/routing-network.yaml b/docker-deployment/config/adapters/routing-network.yaml index 4419664..36820ee 100644 --- a/docker-deployment/config/adapters/routing-network.yaml +++ b/docker-deployment/config/adapters/routing-network.yaml @@ -1,10 +1,14 @@ # Network layer adapter routing. # -# One job: hand discovery to the discovery service, which is a service in this -# same compose and so reachable by name. +# One job: hand the catalogue actions to the discovery service, which is a +# service in this same compose and so reachable by name. # -# The service serves /discover at its root, so no /beckn prefix here: targetType -# "url" appends the action to whatever base is given. +# The service serves /discover and /publish at its root, so no /beckn prefix +# here: targetType "url" appends the action to whatever base is given. +# +# Both actions are one rule because they share a target. publish arrives from +# the provider adapter, discover from the experience adapter, and neither is +# answered here -- this adapter verifies the caller, forwards, and re-signs. routingRules: - version: "2.0.0" targetType: "url" @@ -12,3 +16,4 @@ routingRules: url: "http://discovery:8080" endpoints: - discover + - publish diff --git a/docker-deployment/config/adapters/routing-provider.yaml b/docker-deployment/config/adapters/routing-provider.yaml new file mode 100644 index 0000000..09c2515 --- /dev/null +++ b/docker-deployment/config/adapters/routing-provider.yaml @@ -0,0 +1,20 @@ +# Provider adapter routing -- the OUTBOUND leg, for the /catalog/ module only. +# +# Publishing is the one thing this adapter sends rather than answers: the +# provider's own catalogue system posts here, and the catalogue has to reach +# the network layer, which is what fronts the discovery service. +# +# It is a separate module from /beckn/ because addRoute fails a request whose +# action is not in this file. Sharing one module would mean either listing +# select here -- which would proxy it instead of answering it locally -- or +# breaking select the moment a router was added. +# +# targetType "url" appends the action to the base, so this becomes +# http://network-adapter:9201/beckn/publish. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://network-adapter:9201/beckn" + endpoints: + - publish diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 4c9bdaa..2824c4e 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -198,6 +198,7 @@ services: <<: *adapter-env volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro ports: - "${BIND_ADDR:-127.0.0.1}:${PROVIDER_ADAPTER_PORT}:9200" From 14426dae83705eea546b29b3353625d4fb7bf0c5 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 14:13:22 +0530 Subject: [PATCH 05/81] refactor: mount every adapter on /oan and drop the second module [OpenAgriNet/network-adapter#4] The mount is /oan/ rather than /beckn/, on all three adapters, and the /catalog/ module added for publishing is gone. One subtree per adapter, and the payload's action says which action it is. Dropping that module moved where publishing enters. It cannot enter at the provider adapter any more: the routing step fails an action missing from its config, so routing publish from the module that answers select would mean listing select too, and listing select would proxy it away instead of answering it there. A second mount was what bought both, and a second mount is what has been removed. So publish is a routing entry on the experience adapter instead, which already exists to take a call and forward it. Two entry points now: POST :9202/oan/publish through the experience adapter, which takes unsigned calls from inside the trust boundary and signs on the way out -- so a plain curl works POST :9201/oan/publish straight at the network adapter, for a publisher that signs for itself. Unsigned it answers 401 AUT_SIGNATURE_MISSING. Either way the network adapter verifies the signature and forwards to the discovery service, and the catalogue body still needs no bapId or bppId -- identity travels in the Authorization header's keyId. Config only; no adapter code was touched. Verified on the local stack, all on /oan: select answers on_select with its per-day resources, discover returns catalogs, publish answers ACCEPTED, and an unsigned publish direct to the network adapter is refused 401. --- docker-deployment/README.md | 45 +++++++------- docker-deployment/bin/setup.py | 2 +- .../config/adapters/exp.yaml.tmpl | 2 +- .../config/adapters/network.yaml.tmpl | 2 +- .../config/adapters/provider.yaml.tmpl | 62 +------------------ .../config/adapters/routing-exp.yaml | 10 ++- .../config/adapters/routing-network.yaml | 2 +- .../config/adapters/routing-provider.yaml | 20 ------ docker-deployment/docker-compose.yml | 1 - 9 files changed, 39 insertions(+), 107 deletions(-) delete mode 100644 docker-deployment/config/adapters/routing-provider.yaml diff --git a/docker-deployment/README.md b/docker-deployment/README.md index f070734..29c55c8 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -192,14 +192,14 @@ Replace `my-weather-api` if a different id was used, and point the coordinates at wherever the API has data. ```sh -curl -s -X POST http://127.0.0.1:9202/beckn/select \ +curl -s -X POST http://127.0.0.1:9202/oan/select \ -H 'Content-Type: application/json' \ -d '{ "context": { "version": "2.0.0", "action": "select", "networkId": "oan-dev", - "bapId": "exp.oan.dev", "bapUri": "http://exp-adapter:9202/beckn", - "bppId": "provider.oan.dev", "bppUri": "http://provider-adapter:9200/beckn", + "bapId": "exp.oan.dev", "bapUri": "http://exp-adapter:9202/oan", + "bppId": "provider.oan.dev", "bppUri": "http://provider-adapter:9200/oan", "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", "timestamp": "2026-09-02T06:12:01.330Z" @@ -247,7 +247,7 @@ Three paths, and which adapter answers is the whole design: ``` discover you -> exp -> network -> discovery service select you -> exp -> provider -> your upstream API -publish provider (/catalog/publish) -> network -> discovery service +publish you -> exp -> network -> discovery service ``` `discover` and `publish` both end at the discovery service, and both go @@ -255,25 +255,28 @@ through the network adapter — that adapter is what fronts discovery, verifies the caller and re-signs. `select` never touches it: it goes straight to the provider adapter, which answers from your upstream API. -Publishing enters at the **provider** adapter, on its own mount: +Every adapter mounts one subtree, `/oan/`, and the payload's `action` says +which action it is. There is no per-action path and no second mount. + +Publishing has two entry points, and which one to use depends on whether the +publisher can sign: ```sh -curl -s -X POST http://127.0.0.1:9200/catalog/publish \ +# through the experience adapter -- takes unsigned calls and signs on the way +# out, so this works with a plain curl +curl -s -X POST http://127.0.0.1:9202/oan/publish \ -H 'Content-Type: application/json' -d @your-catalog.json -``` -Two things about that: +# straight at the network adapter -- for a publisher that signs for itself. +# Unsigned, this answers 401 AUT_SIGNATURE_MISSING. +curl -s -X POST http://127.0.0.1:9201/oan/publish ... +``` -- **It is a separate module, `/catalog/`, not another action on `/beckn/`.** - That is forced, not chosen: the routing step fails any request whose action - is missing from its routing config, so a router on `/beckn/` would have to - list `select` as well — and listing it would proxy `select` away instead of - answering it there, which is the one thing that module exists for. -- **The catalogue body needs no `bapId` or `bppId`.** The provider adapter - signs the forwarded request as itself, and the identity travels in the - `Authorization` header's `keyId`, taken from its `keyManager` config. The - network adapter verifies that signature; its identity check skips a body - that declares no caller, so there is nothing to fill in. +**The catalogue body needs no `bapId` or `bppId`.** Identity travels in the +`Authorization` header's `keyId`, taken from the signing adapter's +`keyManager` config. The network adapter verifies that signature, and its +identity check skips a body that declares no caller rather than demanding +one, so there is nothing to fill in. ## The layout @@ -286,9 +289,9 @@ config/ exp.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered provider.yaml.tmpl files hold private keys and are gitignored. - routing-exp.yaml which action goes where. exp splits discover - routing-network.yaml from select; network sends discover and publish - routing-provider.yaml to discovery; provider sends publish onward + routing-exp.yaml which action goes where. exp sends discover and + routing-network.yaml publish to the network layer and select to the + provider; network sends both on to discovery registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. Read at startup -- a change needs the registry diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 76deb85..3e663c8 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -208,7 +208,7 @@ def node(participant_id, name, role, public_key): resolved here: routing between the adapters is the router plugin's config, which uses the compose service names.""" return {"participantId": participant_id, "name": name, "type": "node", - "status": "active", "baseUrl": f"https://{participant_id}/beckn", + "status": "active", "baseUrl": f"https://{participant_id}/oan", "role": role, "keys": signing_key_block(public_key)} diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/docker-deployment/config/adapters/exp.yaml.tmpl index 1b3c3c9..1d13af7 100644 --- a/docker-deployment/config/adapters/exp.yaml.tmpl +++ b/docker-deployment/config/adapters/exp.yaml.tmpl @@ -25,7 +25,7 @@ pluginManager: modules: - name: exp-adapter # A subtree: every action lands here and the payload says which one it is. - path: /beckn/ + path: /oan/ handler: type: std role: bap diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl index a005a7b..88a27f6 100644 --- a/docker-deployment/config/adapters/network.yaml.tmpl +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -23,7 +23,7 @@ pluginManager: modules: - name: network-adapter # A subtree: every action lands here and the payload says which one it is. - path: /beckn/ + path: /oan/ handler: type: std # bpp because this adapter RECEIVES from a BAP. The role decides which diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 06db40e..3f3431b 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -1,6 +1,6 @@ # OAN provider adapter -- dev deployment. # -# Serves /beckn/ synchronously: verifies the sender against the registry, +# Serves /oan/ synchronously: verifies the sender against the registry, # resolves the capability's call plan, calls the provider, and answers with the # mapped result. No callback -- the answer is the HTTP response. appName: "oan-provider-adapter" @@ -24,9 +24,9 @@ pluginManager: modules: - name: oanProvider # A subtree, not one action. Without the trailing slash Go's ServeMux - # matches exactly, so /beckn/select would mount that action and 404 the + # matches exactly, so /oan/select would mount that action and 404 the # rest. Which action it is comes from the payload, never the URL. - path: /beckn/ + path: /oan/ handler: type: std role: bpp @@ -96,59 +96,3 @@ modules: - validateSign # the sender's key, from the registry - weather # resolve, map out, call, map back - signAck # signs whatever the step answered with - # The outbound leg: publishing a catalogue to the network layer. - # - # A SECOND module, not another action on /beckn/, and that is forced rather - # than chosen. addRoute fails any request whose action is missing from its - # routing config, so a router on /beckn/ would have to list select too -- - # and listing select would proxy it away instead of answering it here, which - # is the one thing that module exists to do. - # - # POST /catalog/publish. No validateSign: the caller is the provider's own - # catalogue system, inside its trust boundary, exactly as the experience - # adapter takes unsigned calls from the app in front of it. - # - # Nothing here needs bapId or bppId in the body. This adapter signs the - # forwarded request as itself, and the Authorization header's keyId carries - # the identity -- taken from keyManager below, not from the payload. The - # network layer's identity check skips a body that declares no caller. - - name: oanProviderPublish - path: /catalog/ - handler: - type: std - # bap because this module SENDS. The role decides which context identity - # is compared against a signer, and nothing is verified on the way in - # here -- but the same role picks the identity used on the way out. - role: bap - subscriberId: __PROVIDER_SUBSCRIBER_ID__ - - plugins: - registry: - id: oanregistry - config: - url: http://registry:8081/api/v1 - entity: Participant - providerEntity: ProviderSchema - - keyManager: - id: simplekeymanager - config: - subscriberId: __PROVIDER_SUBSCRIBER_ID__ - keyId: __PROVIDER_KEY_ID__ - signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" - signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" - encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" - encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" - - signer: - id: signer - - router: - id: router - config: - routingConfig: /app/config/routing-provider.yaml - - steps: - - addRoute # publish -> the network layer - - sign # as this provider, so the network layer can verify it - diff --git a/docker-deployment/config/adapters/routing-exp.yaml b/docker-deployment/config/adapters/routing-exp.yaml index d759e9c..fbe062d 100644 --- a/docker-deployment/config/adapters/routing-exp.yaml +++ b/docker-deployment/config/adapters/routing-exp.yaml @@ -10,14 +10,20 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://network-adapter:9201/beckn" + url: "http://network-adapter:9201/oan" endpoints: - discover + # Publishing a catalogue also goes to the network layer, which is what + # fronts the discovery service. It enters here so a publisher does not + # have to sign for itself: this adapter takes unsigned calls from inside + # the trust boundary and signs on the way out. A publisher that can sign + # may post to the network adapter's /oan/publish directly instead. + - publish - version: "2.0.0" targetType: "url" target: - url: "http://provider-adapter:9200/beckn" + url: "http://provider-adapter:9200/oan" endpoints: - select - init diff --git a/docker-deployment/config/adapters/routing-network.yaml b/docker-deployment/config/adapters/routing-network.yaml index 36820ee..41b0723 100644 --- a/docker-deployment/config/adapters/routing-network.yaml +++ b/docker-deployment/config/adapters/routing-network.yaml @@ -3,7 +3,7 @@ # One job: hand the catalogue actions to the discovery service, which is a # service in this same compose and so reachable by name. # -# The service serves /discover and /publish at its root, so no /beckn prefix +# The service serves /discover and /publish at its root, so no /oan prefix # here: targetType "url" appends the action to whatever base is given. # # Both actions are one rule because they share a target. publish arrives from diff --git a/docker-deployment/config/adapters/routing-provider.yaml b/docker-deployment/config/adapters/routing-provider.yaml deleted file mode 100644 index 09c2515..0000000 --- a/docker-deployment/config/adapters/routing-provider.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Provider adapter routing -- the OUTBOUND leg, for the /catalog/ module only. -# -# Publishing is the one thing this adapter sends rather than answers: the -# provider's own catalogue system posts here, and the catalogue has to reach -# the network layer, which is what fronts the discovery service. -# -# It is a separate module from /beckn/ because addRoute fails a request whose -# action is not in this file. Sharing one module would mean either listing -# select here -- which would proxy it instead of answering it locally -- or -# breaking select the moment a router was added. -# -# targetType "url" appends the action to the base, so this becomes -# http://network-adapter:9201/beckn/publish. -routingRules: - - version: "2.0.0" - targetType: "url" - target: - url: "http://network-adapter:9201/beckn" - endpoints: - - publish diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 2824c4e..4c9bdaa 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -198,7 +198,6 @@ services: <<: *adapter-env volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro - - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro ports: - "${BIND_ADDR:-127.0.0.1}:${PROVIDER_ADAPTER_PORT}:9200" From ef6f41c9b8fb211eada3c516e80d187c04b96310 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 14:17:51 +0530 Subject: [PATCH 06/81] feat: publish at the provider adapter, which signs and forwards [OpenAgriNet/network-adapter#4] The provider's own catalogue system posts POST /publish to the provider adapter. That adapter signs the request as itself and forwards it to the network layer, which verifies the signature and hands it to the discovery service. The caller signs nothing. It is a second module on that adapter, and it has to be: the routing step fails any action missing from its config, so routing publish from the module that answers select would mean listing select too -- and listing select would proxy it to the network layer instead of answering it there. The new module sits at the root rather than under /oan/, and that split is the point. /oan/ is the adapter's Beckn surface -- the path the registry publishes as its baseUrl -- so a peer calling /select has to reach the module that answers select. Publishing is not part of that surface; it arrives from inside the provider's own deployment. Go's mux takes the longest matching pattern, so /oan/... still reaches the capability module and only /publish falls through to the root one. No prefix is invented for either. publish is no longer a routing entry on the experience adapter: it enters at the provider now, so exp is back to discover and the transactional actions. Config only; no adapter code was touched. Verified on the local stack: POST /publish answers ACCEPTED with the catalogue's stats, select still answers on_select with its per-day resources, discover still returns catalogs, and an unsigned POST /oan/select at the provider adapter is refused 401 -- which is what proves the two mounts are resolving to the right modules rather than the root one swallowing both. --- docker-deployment/README.md | 49 ++++++++------ .../config/adapters/provider.yaml.tmpl | 64 +++++++++++++++++++ .../config/adapters/routing-exp.yaml | 6 -- .../config/adapters/routing-provider.yaml | 15 +++++ docker-deployment/docker-compose.yml | 1 + 5 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 docker-deployment/config/adapters/routing-provider.yaml diff --git a/docker-deployment/README.md b/docker-deployment/README.md index 29c55c8..f221ae6 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -247,7 +247,7 @@ Three paths, and which adapter answers is the whole design: ``` discover you -> exp -> network -> discovery service select you -> exp -> provider -> your upstream API -publish you -> exp -> network -> discovery service +publish your catalogue system -> provider -> network -> discovery service ``` `discover` and `publish` both end at the discovery service, and both go @@ -255,28 +255,36 @@ through the network adapter — that adapter is what fronts discovery, verifies the caller and re-signs. `select` never touches it: it goes straight to the provider adapter, which answers from your upstream API. -Every adapter mounts one subtree, `/oan/`, and the payload's `action` says -which action it is. There is no per-action path and no second mount. +Each adapter's Beckn surface is one subtree, `/oan/`, and the payload's +`action` says which action it is. That is the path the registry publishes as +a participant's `baseUrl`, so a peer calling `/select` lands on the +module that answers select. -Publishing has two entry points, and which one to use depends on whether the -publisher can sign: +Publishing enters at the **provider** adapter, which signs and forwards: ```sh -# through the experience adapter -- takes unsigned calls and signs on the way -# out, so this works with a plain curl -curl -s -X POST http://127.0.0.1:9202/oan/publish \ +curl -s -X POST http://127.0.0.1:9200/publish \ -H 'Content-Type: application/json' -d @your-catalog.json - -# straight at the network adapter -- for a publisher that signs for itself. -# Unsigned, this answers 401 AUT_SIGNATURE_MISSING. -curl -s -X POST http://127.0.0.1:9201/oan/publish ... ``` -**The catalogue body needs no `bapId` or `bppId`.** Identity travels in the -`Authorization` header's `keyId`, taken from the signing adapter's -`keyManager` config. The network adapter verifies that signature, and its -identity check skips a body that declares no caller rather than demanding -one, so there is nothing to fill in. +Three things about that: + +- **`/publish` sits outside `/oan/`, at the root.** It is not part of this + adapter's Beckn surface — it arrives from inside the provider's own + deployment — so it must not shadow it. Go's mux takes the longest matching + pattern, so `/oan/select` still reaches the capability module and only + `/publish` falls to the root one. +- **It is a second module, and has to be.** The routing step fails any action + missing from its config, so routing publish from the module that answers + select would mean listing select too — and listing select would proxy it to + the network layer instead of answering it there. +- **The catalogue body needs no `bapId` or `bppId`, and the caller need not + sign.** The provider's own catalogue system is inside its trust boundary, + so this module verifies nothing on the way in; it signs the forwarded + request as itself, and identity travels in the `Authorization` header's + `keyId` from its `keyManager` config. The network adapter verifies that + signature — and its identity check skips a body that declares no caller + rather than demanding one. ## The layout @@ -289,9 +297,10 @@ config/ exp.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered provider.yaml.tmpl files hold private keys and are gitignored. - routing-exp.yaml which action goes where. exp sends discover and - routing-network.yaml publish to the network layer and select to the - provider; network sends both on to discovery + routing-exp.yaml which action goes where. exp sends discover to + routing-network.yaml the network layer and select to the provider; + routing-provider.yaml provider sends publish to the network layer; + network sends discover and publish to discovery registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. Read at startup -- a change needs the registry diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 3f3431b..16fc151 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -96,3 +96,67 @@ modules: - validateSign # the sender's key, from the registry - weather # resolve, map out, call, map back - signAck # signs whatever the step answered with + + # The outbound leg: the provider's own catalogue system publishing to the + # network layer. POST /publish. + # + # Mounted at the root, and /oan/ above wins for anything under it because + # Go's mux takes the longest matching pattern. That split is deliberate: + # /oan/ is this adapter's Beckn surface, the path the registry publishes as + # its baseUrl, so a network peer calling /select has to land on the + # module that answers select. Publishing is not part of that surface -- it + # arrives from inside this provider's own deployment -- so it sits outside + # it rather than shadowing it. + # + # It has to be a second module, not another action on /oan/: the routing + # step fails any action missing from its config, so routing publish from + # the module that answers select would mean listing select too -- and + # listing select would proxy it to the network layer instead of answering + # it here, which is the one thing that module does. + # + # No validateSign: the caller is this provider's own catalogue system, + # inside its trust boundary, exactly as the experience adapter takes + # unsigned calls from the app in front of it. This adapter then signs the + # forwarded request as itself, which is what the network layer verifies. + # + # Nothing here needs bapId or bppId in the body. Identity travels in the + # Authorization header's keyId, taken from keyManager below. + - name: oanProviderPublish + path: / + handler: + type: std + # bap because this module SENDS. + role: bap + subscriberId: __PROVIDER_SUBSCRIBER_ID__ + + plugins: + registry: + id: oanregistry + config: + url: http://registry:8081/api/v1 + entity: Participant + providerEntity: ProviderSchema + + keyManager: + id: simplekeymanager + config: + subscriberId: __PROVIDER_SUBSCRIBER_ID__ + keyId: __PROVIDER_KEY_ID__ + signingPrivateKey: "__PROVIDER_SIGNING_PRIVATE__" + signingPublicKey: "__PROVIDER_SIGNING_PUBLIC__" + encrPrivateKey: "__PROVIDER_ENCR_PRIVATE__" + encrPublicKey: "__PROVIDER_ENCR_PUBLIC__" + + signer: + id: signer + + router: + id: router + config: + routingConfig: /app/config/routing-provider.yaml + + steps: + - addRoute # publish -> the network layer + - sign # as this provider, so the network layer can verify + + diff --git a/docker-deployment/config/adapters/routing-exp.yaml b/docker-deployment/config/adapters/routing-exp.yaml index fbe062d..1a9c2f9 100644 --- a/docker-deployment/config/adapters/routing-exp.yaml +++ b/docker-deployment/config/adapters/routing-exp.yaml @@ -13,12 +13,6 @@ routingRules: url: "http://network-adapter:9201/oan" endpoints: - discover - # Publishing a catalogue also goes to the network layer, which is what - # fronts the discovery service. It enters here so a publisher does not - # have to sign for itself: this adapter takes unsigned calls from inside - # the trust boundary and signs on the way out. A publisher that can sign - # may post to the network adapter's /oan/publish directly instead. - - publish - version: "2.0.0" targetType: "url" diff --git a/docker-deployment/config/adapters/routing-provider.yaml b/docker-deployment/config/adapters/routing-provider.yaml new file mode 100644 index 0000000..bb9dd0d --- /dev/null +++ b/docker-deployment/config/adapters/routing-provider.yaml @@ -0,0 +1,15 @@ +# Provider adapter routing -- for the root module only. +# +# Publishing is the one thing this adapter sends rather than answers. The +# provider's own catalogue system posts POST /publish here, and the catalogue +# has to reach the network layer, which is what fronts the discovery service. +# +# targetType "url" appends the action to the base, so this becomes +# http://network-adapter:9201/oan/publish. +routingRules: + - version: "2.0.0" + targetType: "url" + target: + url: "http://network-adapter:9201/oan" + endpoints: + - publish diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 4c9bdaa..2824c4e 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -198,6 +198,7 @@ services: <<: *adapter-env volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro ports: - "${BIND_ADDR:-127.0.0.1}:${PROVIDER_ADAPTER_PORT}:9200" From 280a2ec7965ec8baa903d90039c402beec19f475 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 14:23:20 +0530 Subject: [PATCH 07/81] feat: name no party in a payload, and pull the published adapter image [OpenAgriNet/network-adapter#4] bapId, bppId, bapUri and bppUri are gone from every payload -- the example select and discover bodies, the catalogue body, and the prose that described them as what goes on the wire. Identity travels in the Authorization header's keyId, which names the signer and the key the registry published for it, and a body that declares no caller skips the declared-identity comparison rather than failing it. The two *Uri fields were container-internal addresses that meant nothing outside the compose network in any case. Catalogue-level bppId and bppUri go too. The discovery service never names them -- they ride through in the stored document -- so publishing without them is accepted and discover still returns the catalogue. ADAPTER_IMAGE is ghcr.io/nisargabd/oan-adapter:latest. Verified pullable anonymously, and verified to carry the three plugins these configs name: oanregistry.so, jsonmapper.so and weather.so, alongside router, signer, signvalidator and simplekeymanager. DISCOVERY_IMAGE is still unset -- no tag has been shared for it yet. Verified on the local stack with no party named anywhere: publish answers ACCEPTED, select answers on_select with its per-day resources and a context carrying only correlation ids, and discover returns the catalogue. --- docker-deployment/.env.example | 6 +- docker-deployment/README.md | 16 +- docker-deployment/bin/setup.py | 2 +- .../config/adapters/network.yaml.tmpl | 9 +- .../config/adapters/provider.yaml.tmpl | 5 +- .../config/registry/schemas/Participant.json | 353 +++++++++++++++++- 6 files changed, 356 insertions(+), 35 deletions(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index 80f65d6..5909b95 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -26,7 +26,7 @@ BIND_ADDR=127.0.0.1 # # If they live in a private registry, log in on the VM first: # docker login ghcr.io -ADAPTER_IMAGE=REPLACE_ME +ADAPTER_IMAGE=ghcr.io/nisargabd/oan-adapter:latest DISCOVERY_IMAGE=REPLACE_ME # ---- ports ----------------------------------------------------------------- @@ -62,8 +62,8 @@ BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v # bin/setup.py registers exactly these three in the registry and generates a # keypair for each. # -# A participant id IS the network identity -- what goes on the wire as -# context.bapId / bppId -- so the registry requires it to be hostname-shaped. +# A participant id IS the network identity -- the id a signature is verified +# against -- so the registry requires it to be hostname-shaped. # They are never resolved by DNS: routing between the adapters is the router # plugin's config, which uses the compose service names. # diff --git a/docker-deployment/README.md b/docker-deployment/README.md index f221ae6..a299503 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -198,8 +198,6 @@ curl -s -X POST http://127.0.0.1:9202/oan/select \ "context": { "version": "2.0.0", "action": "select", "networkId": "oan-dev", - "bapId": "exp.oan.dev", "bapUri": "http://exp-adapter:9202/oan", - "bppId": "provider.oan.dev", "bppUri": "http://provider-adapter:9200/oan", "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", "timestamp": "2026-09-02T06:12:01.330Z" @@ -227,14 +225,12 @@ curl -s -X POST http://127.0.0.1:9202/oan/select \ You should get an `on_select` back, with one resource per forecast day. -Two things about identity here: - -- The **request** carries `bapId` / `bppId`, because that is the caller saying - who it is. The adapter needs it to look the sender's key up in the registry. -- The **answer** does not echo them back. A mapping transforms a payload; it - does not assert who anyone is, and the `*Uri` fields it could copy are - container-internal addresses that mean nothing outside this compose network. - Identity on the answer is the adapter's signature over it. +No party is named in the payload, in either direction. Identity travels in +the `Authorization` header's `keyId`, which names the signer and the key the +registry published for it; a body that declares no caller simply skips the +declared-identity comparison. Nothing needs `bapId` or `bppId`, and the +`*Uri` fields they came with were container-internal addresses that meant +nothing outside this compose network anyway. The experience adapter is the only one that takes an unsigned request — the experience app is inside the trust boundary, so there is no network signature diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 3e663c8..16f3794 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -204,7 +204,7 @@ def node(participant_id, name, role, public_key): One level, no wrapper object: type decides which fields apply. baseUrl must be https for a node, and the id must be hostname-shaped -- it is the - identity that goes on the wire as context.bapId / bppId. Neither is + identity a signature is checked against. Neither is resolved here: routing between the adapters is the router plugin's config, which uses the compose service names.""" return {"participantId": participant_id, "name": name, "type": "node", diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl index 88a27f6..0a692e8 100644 --- a/docker-deployment/config/adapters/network.yaml.tmpl +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -26,10 +26,11 @@ modules: path: /oan/ handler: type: std - # bpp because this adapter RECEIVES from a BAP. The role decides which - # context identity validateSign compares the signer against: bap expects - # the sender to be the bppId, bpp expects the bapId -- and the sender here - # is the caller, oan-caller. + # bpp because this adapter RECEIVES rather than originates. The role + # decides which declared identity validateSign would compare a signer + # against -- but no payload here declares one, so that check is skipped + # and what is verified is the signature itself, against the key the + # registry publishes for whoever signed. role: bpp subscriberId: __NETWORK_SUBSCRIBER_ID__ diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 16fc151..0253c08 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -119,8 +119,9 @@ modules: # unsigned calls from the app in front of it. This adapter then signs the # forwarded request as itself, which is what the network layer verifies. # - # Nothing here needs bapId or bppId in the body. Identity travels in the - # Authorization header's keyId, taken from keyManager below. + # No party is named in the body at all. Identity travels in the + # Authorization header's keyId, taken from keyManager below, and that is + # what the network layer verifies against the registry. - name: oanProviderPublish path: / handler: diff --git a/docker-deployment/config/registry/schemas/Participant.json b/docker-deployment/config/registry/schemas/Participant.json index 6d651bc..ab04c8f 100644 --- a/docker-deployment/config/registry/schemas/Participant.json +++ b/docker-deployment/config/registry/schemas/Participant.json @@ -40,22 +40,22 @@ "$ref": "#/definitions/Status" }, "baseUrl": { - "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https for a node, since that is its wire identity; an upstream may be plaintext, because nothing in the registry is sent to it.", + "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https, except that an upstream with auth.scheme 'none' may be plaintext.", "type": "string", "maxLength": 2000, "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" }, "role": { - "description": "What this party does on the network. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs.", + "description": "What this node does on the network. BAP asks. BPP answers. NETWORK exposes publish and discover, answering discover from published catalogs.", "type": "string", "enum": [ - "consumer", - "provider", - "network" + "BAP", + "BPP", + "NETWORK" ] }, "keys": { - "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header.", + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used in the Authorization keyId.", "type": "array", "minItems": 1, "maxItems": 8, @@ -63,11 +63,14 @@ "items": { "$ref": "#/definitions/PublicKey" } + }, + "auth": { + "$ref": "#/definitions/Auth" } }, "allOf": [ { - "description": "A node speaks Beckn: it needs a role and keys, and its id is its wire identity so it must be a hostname.", + "description": "A node speaks Beckn: it needs a role and keys, has no credential of ours to present, and its id is its wire identity so it must be a hostname.", "if": { "properties": { "type": { @@ -83,6 +86,15 @@ "role", "keys" ], + "not": { + "anyOf": [ + { + "required": [ + "auth" + ] + } + ] + }, "properties": { "participantId": { "maxLength": 253, @@ -95,7 +107,7 @@ } }, { - "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify.", + "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify, only a credential we present.", "if": { "properties": { "type": { @@ -107,6 +119,9 @@ ] }, "then": { + "required": [ + "auth" + ], "not": { "anyOf": [ { @@ -122,22 +137,66 @@ ] } } + }, + { + "description": "A credential over plaintext http is a leaked credential.", + "if": { + "properties": { + "auth": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "not": { + "const": "none" + } + } + } + } + }, + "required": [ + "auth" + ] + }, + "then": { + "properties": { + "baseUrl": { + "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" + } + } + } } ] }, "PublicKey": { - "description": "A public key, held as material: it is public, so there is nothing to protect. Identified by the osid the registry assigns on write -- which is what a sender names in the Authorization header, and so the only id a verifier can look up. alg carries the purpose: ed25519 signs, x25519 encrypts.", + "description": "A public key, held as material: it is public, so there is nothing to protect. Contrast Secret, always a pointer.", "type": "object", "additionalProperties": false, "required": [ + "keyId", + "use", "alg", "key", "validFrom", "status" ], "properties": { + "keyId": { + "description": "Second field of the Authorization keyId, so a sender can say which key it signed with.", + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" + }, + "use": { + "description": "sign verifies signatures; encrypt is for encrypted payloads.", + "type": "string", + "enum": [ + "sign", + "encrypt" + ] + }, "alg": { - "description": "The purpose, and the only thing that carries it now: ed25519 signs, x25519 encrypts. Both curves are 32-byte keys.", + "description": "Fixed by use. Both curves are 32-byte keys.", "type": "string", "enum": [ "ed25519", @@ -145,9 +204,9 @@ ] }, "key": { - "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. Bare -- no encoding label in front of it. A truncated or wrong-curve key fails at write time.", + "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. A truncated or wrong-curve key fails at write time.", "type": "string", - "pattern": "^[A-Za-z0-9+/]{43}=$" + "pattern": "^base64:[A-Za-z0-9+/]{43}=$" }, "validFrom": { "type": "string", @@ -166,10 +225,263 @@ "revoked" ] } - } + }, + "allOf": [ + { + "if": { + "properties": { + "use": { + "const": "sign" + } + } + }, + "then": { + "properties": { + "alg": { + "const": "ed25519" + } + } + } + }, + { + "if": { + "properties": { + "use": { + "const": "encrypt" + } + } + }, + "then": { + "properties": { + "alg": { + "const": "x25519" + } + } + } + } + ] + }, + "Auth": { + "description": "How our adapter authenticates TO an upstream. Not Beckn signing \u2014 that uses Node.keys.", + "type": "object", + "additionalProperties": false, + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "type": "string", + "enum": [ + "none", + "apiKeyQuery", + "apiKeyHeader", + "basic" + ] + }, + "paramName": { + "description": "The single query-parameter or header name the credential goes in.", + "$ref": "#/definitions/ParamName" + }, + "valuePrefix": { + "description": "Prepended to the credential, trailing space included \u2014 'Bearer '. Header schemes only.", + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,30} $" + }, + "paramNames": { + "description": "For an upstream wanting several named credentials. Keys must match `secrets` exactly.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/definitions/ParamName" + } + }, + "secrets": { + "description": "Pointers to credentials held outside the registry. Redacted from /search by _osConfig.privateFields.", + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/definitions/Secret" + } + } + }, + "allOf": [ + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "const": "none" + } + } + }, + "then": { + "allOf": [ + { + "not": { + "required": [ + "secrets" + ] + } + }, + { + "not": { + "required": [ + "paramName" + ] + } + }, + { + "not": { + "required": [ + "paramNames" + ] + } + }, + { + "not": { + "required": [ + "valuePrefix" + ] + } + } + ] + } + }, + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "enum": [ + "apiKeyQuery", + "apiKeyHeader" + ] + } + } + }, + "then": { + "required": [ + "secrets" + ], + "oneOf": [ + { + "required": [ + "paramName" + ], + "not": { + "required": [ + "paramNames" + ] + }, + "properties": { + "secrets": { + "maxProperties": 1 + } + } + }, + { + "required": [ + "paramNames" + ], + "allOf": [ + { + "not": { + "required": [ + "paramName" + ] + } + }, + { + "not": { + "required": [ + "valuePrefix" + ] + } + } + ] + } + ] + } + }, + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "const": "apiKeyQuery" + } + } + }, + "then": { + "not": { + "required": [ + "valuePrefix" + ] + } + } + }, + { + "if": { + "required": [ + "scheme" + ], + "properties": { + "scheme": { + "const": "basic" + } + } + }, + "then": { + "required": [ + "secrets" + ], + "properties": { + "secrets": { + "required": [ + "username", + "password" + ] + } + }, + "allOf": [ + { + "not": { + "required": [ + "paramName" + ] + } + }, + { + "not": { + "required": [ + "paramNames" + ] + } + }, + { + "not": { + "required": [ + "valuePrefix" + ] + } + } + ] + } + } + ] + }, + "ParamName": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" }, "ParticipantId": { - "description": "Stable id, and the only id. For a node this is its network identity (context.bapId / context.bppId); for an upstream it is the Beckn offer.provider.id.", + "description": "Stable id, and the only id. For a node this is its network identity -- what field 1 of the Authorization header names, and what a signature is verified against; for an upstream it is the Beckn offer.provider.id.", "type": "string", "maxLength": 253, "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" @@ -181,6 +493,15 @@ "inactive" ] }, + "MaterialRef": { + "description": "A pointer to SECRET material held outside the registry, :. Never the material. Public keys are not MaterialRefs: PublicKey.key holds the bytes.", + "type": "string", + "maxLength": 1024, + "pattern": "^(env://[A-Z][A-Z0-9_]{0,63}|inline:[!-~][ -~]{0,998})$" + }, + "Secret": { + "$ref": "#/definitions/MaterialRef" + }, "ParticipantType": { "description": "node speaks Beckn. upstream is an API we call over ordinary HTTP and does not.", "type": "string", @@ -199,7 +520,9 @@ "type", "baseUrl" ], - "privateFields": [], + "privateFields": [ + "$.auth.secrets" + ], "roles": [ "registryOperator" ], From b8025a3e5b1b5ffc073891028953d0c4590dc2b3 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 14:30:12 +0530 Subject: [PATCH 08/81] feat: validate payloads against the pinned Beckn v2 spec [OpenAgriNet/network-adapter#4] Every adapter config declares schemav2validator, pointed at the LTS spec tag, with the extended layer off: it fetches each resource's own @context and validates against that, which is a network call per payload and a second failure mode, and nothing here needs it. The extendedSchema_* settings are present but inert until it is switched on. The validateSchema step is wired only where every action the module handles is defined in that spec: exp /oan/ discover, select, init, confirm, status -- validated provider /oan/ select -- validated network /oan/ carries publish -- declared, not wired provider / carries publish -- declared, not wired publish is absent from the spec, and the validator refuses an action it cannot find rather than passing it through, so wiring the step on either of those modules would reject every catalogue with "unsupported action: publish". Validating it needs an auxiliary spec that defines the action -- auxiliaryTypes and auxiliaryLocations, which are additive and must not overlap the primary spec. Turning validation on surfaced a defect in the spec itself, and the example payloads now carry a quantity because of it: Commitment.resources requires ["id", "quantity"] while Resource defines no quantity property and the spec has no Quantity schema anywhere. additionalProperties is unset, so any value satisfies it. Without one every select is refused SCH_REQUIRED_FIELD_MISSING. That is upstream, not a choice made here, and the README says so where someone writing a payload will read it. Verified on the local stack: select, discover and publish all answer 200 with validation on; a select missing quantity is refused 400 SCH_REQUIRED_FIELD_MISSING naming the JSON path, an unknown action is refused 400 "unsupported action", and extendedSchema_enabled reads "false" inside the running container. --- docker-deployment/README.md | 28 +++++++++++++++ .../config/adapters/exp.yaml.tmpl | 18 ++++++++++ .../config/adapters/network.yaml.tmpl | 17 +++++++++ .../config/adapters/provider.yaml.tmpl | 35 +++++++++++++++++++ 4 files changed, 98 insertions(+) diff --git a/docker-deployment/README.md b/docker-deployment/README.md index a299503..b372835 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -206,6 +206,7 @@ curl -s -X POST http://127.0.0.1:9202/oan/select \ "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, "resources": [ { "id": "res:point-forecast", + "quantity": 1, "resourceAttributes": { "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", @@ -282,6 +283,33 @@ Three things about that: signature — and its identity check skips a body that declares no caller rather than demanding one. +## Schema validation + +Every adapter loads the pinned Beckn v2 LTS spec and validates request bodies +against it. The **extended** layer is off: it fetches each resource's own +`@context` and validates against that, which is a network call per payload and +a second thing that can fail. The `extendedSchema_*` settings in the configs +only take effect if it is switched on. + +Two consequences worth knowing before you write a payload: + +- **Each resource under a commitment needs a `quantity`.** The spec's + `Commitment.resources` requires `["id", "quantity"]` while `Resource` itself + defines no `quantity` property and the spec has no `Quantity` schema at all — + a defect upstream, not something this deployment chose. Any value satisfies + it. Without one, every `select` is refused with + `SCH_REQUIRED_FIELD_MISSING: property "quantity" is missing`. +- **`publish` is not validated, because the spec does not define it.** The + validator refuses an action it cannot find with `unsupported action: publish`, + so the two modules that carry publishing — the provider adapter's root mount + and the network adapter — declare the validator but do not run it. To + validate publishing, give the validator an auxiliary spec that defines the + action: `auxiliaryTypes` and `auxiliaryLocations`, which are additive and + must not overlap the primary spec. + +An action the spec does not know, or a body missing a required field, comes +back as a signed NACK with a `SCH_*` code and the JSON path that failed. + ## The layout ``` diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/docker-deployment/config/adapters/exp.yaml.tmpl index 1d13af7..94e0c21 100644 --- a/docker-deployment/config/adapters/exp.yaml.tmpl +++ b/docker-deployment/config/adapters/exp.yaml.tmpl @@ -56,11 +56,29 @@ modules: signValidator: id: signvalidator + # Base Beckn v2 schema validation, against the pinned LTS spec. The + # extended layer is off: it fetches a resource's own @context and + # validates against that, which is a network call per payload and a + # second failure mode, and nothing here needs it yet. The allowed + # domains and cache settings below only take effect if it is turned on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + router: id: router config: routingConfig: /app/config/routing-exp.yaml steps: + - validateSchema - addRoute - sign diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl index 0a692e8..2086c7e 100644 --- a/docker-deployment/config/adapters/network.yaml.tmpl +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -59,6 +59,23 @@ modules: signValidator: id: signvalidator + # Base Beckn v2 schema validation, against the pinned LTS spec. The + # extended layer is off: it fetches a resource's own @context and + # validates against that, which is a network call per payload and a + # second failure mode, and nothing here needs it yet. The allowed + # domains and cache settings below only take effect if it is turned on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + router: id: router config: diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 0253c08..5d90e4a 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -63,6 +63,23 @@ modules: signer: id: signer + # Base Beckn v2 schema validation, against the pinned LTS spec. The + # extended layer is off: it fetches a resource's own @context and + # validates against that, which is a network call per payload and a + # second failure mode, and nothing here needs it yet. The allowed + # domains and cache settings below only take effect if it is turned on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + # Generic: fetches, compiles and caches whatever the registry's mapping # URLs point at. Knows nothing about any provider. mapper: @@ -94,6 +111,7 @@ modules: steps: - validateSign # the sender's key, from the registry + - validateSchema # the pinned Beckn v2 spec - weather # resolve, map out, call, map back - signAck # signs whatever the step answered with @@ -151,6 +169,23 @@ modules: signer: id: signer + # Base Beckn v2 schema validation, against the pinned LTS spec. The + # extended layer is off: it fetches a resource's own @context and + # validates against that, which is a network call per payload and a + # second failure mode, and nothing here needs it yet. The allowed + # domains and cache settings below only take effect if it is turned on. + schemaValidator: + id: schemav2validator + config: + type: url + location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema_enabled: "false" + extendedSchema_cacheTTL: "86400" + extendedSchema_maxCacheSize: "100" + extendedSchema_downloadTimeout: "30" + extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + router: id: router config: From dbef6f8efa0b4946b5dddfb03df0243a0e0244cb Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 14:40:50 +0530 Subject: [PATCH 09/81] fix: name the discovery image, and repair the bring-up sequence [OpenAgriNet/network-adapter#4] DISCOVERY_IMAGE is ghcr.io/nisargabd/discovery-service:${TAG:-latest}, with TAG pinning a known build and unset meaning latest. Compose interpolates a .env value, so both resolve: verified :latest with TAG unset and :v0.3.1 with TAG set. With both images published the stack was runnable for the first time, and running it end to end found two things. THE DOCUMENTED BRING-UP WAS BROKEN. Step one was a bare `docker compose up -d`, which starts the adapters too -- and an adapter config is a bind-mounted FILE that step two has not written yet, so Docker created a DIRECTORY at each of those paths. The adapters then died on "adapter.yaml: is a directory", the later `up` did not recreate them so they stayed dead, and the directories sat where setup.py needed to write files. Step one now names registry and discovery explicitly, the compose header and README say why, and setup.py refuses with an explanation and the commands to recover if it finds a directory where a config belongs. THE PARTICIPANT SCHEMA HAD BEEN REVERTED. Porting a description reword in 20ccc9d copied oan-local's copy over this one, which still carries the old contract, undoing 37f02c6: the role enum was back to BAP/BPP/NETWORK, keyId and use were required again, keys wanted the base64: label and auth was back. Re-applied, with the reworded descriptions kept. oan-local is deliberately still on the old contract, so it is not a source to copy this file from. Verified from an empty volume, following the README exactly, on the published images: the three adapter rows seed, all eight services come up, the two provider rows register, and then select answers on_select with three per-day resources and no dangling offer references, publish answers ACCEPTED, and discover returns the catalogue. --- docker-deployment/.env.example | 5 +- docker-deployment/README.md | 23 +- docker-deployment/bin/setup.py | 12 + .../config/registry/schemas/Participant.json | 351 +----------------- docker-deployment/docker-compose.yml | 18 +- 5 files changed, 58 insertions(+), 351 deletions(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index 5909b95..9d88e54 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -26,8 +26,11 @@ BIND_ADDR=127.0.0.1 # # If they live in a private registry, log in on the VM first: # docker login ghcr.io +# TAG pins the discovery service; unset means latest. Set it to deploy a known +# build rather than whatever latest points at today: +# TAG=v0.3.1 docker compose up -d ADAPTER_IMAGE=ghcr.io/nisargabd/oan-adapter:latest -DISCOVERY_IMAGE=REPLACE_ME +DISCOVERY_IMAGE=ghcr.io/nisargabd/discovery-service:${TAG:-latest} # ---- ports ----------------------------------------------------------------- REGISTRY_PORT=8081 diff --git a/docker-deployment/README.md b/docker-deployment/README.md index b372835..45d9de9 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -52,7 +52,6 @@ On the VM: - Docker with Compose v2, logged in to wherever the images live if it is private — `docker login ghcr.io` -- the image tags for the adapter and the discovery service - Python 3 and the `cryptography` package — `pip install cryptography` And a URL the VM can reach for the upstream provider API. If that API runs on @@ -67,8 +66,10 @@ cp .env.example .env Read `.env` before going on. Four things in it matter: -- `ADAPTER_IMAGE` and `DISCOVERY_IMAGE` — the tags to pull. No working default; - set them to the tags published for this environment. +- `TAG` — pins the discovery service. Unset means `latest`; set it to deploy a + known build instead of whatever `latest` points at today: + `TAG=v0.3.1 docker compose up -d`. The images themselves are already named in + `.env.example` and are pulled, never built. - **the credentials.** All shipped defaults. Change them. - `BIND_ADDR` — see above. - `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` have to match the registry @@ -77,19 +78,25 @@ Read `.env` before going on. Four things in it matter: Then: ```sh -# 1. pulls the images, then starts registry and discovery. The adapters will -# restart in a loop for now -- their configs do not exist yet, which step -# 2 fixes. -docker compose up -d +# 1. everything EXCEPT the adapters. Their configs do not exist yet, and +# step 2 is what writes them. +docker compose up -d registry discovery # 2. generate the adapter keypairs, register the three adapter identities, # render the three adapter configs python3 bin/setup.py -# 3. now the adapters have configs to read +# 3. now the adapters docker compose up -d ``` +**Do not run a bare `docker compose up -d` for step 1.** An adapter config is +a bind-mounted *file*, and Docker creates a *directory* at any bind-mount +source that is missing. Starting an adapter early therefore wedges it on a +directory it cannot parse — `adapter.yaml: is a directory` — and leaves a +directory where step 2 needs to write a file. `bin/setup.py` refuses with an +explanation if it finds one; delete the empty directories and re-run. + Check it: ```sh diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 16f3794..2a3b323 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -292,6 +292,18 @@ def render(identities): if "__" in template: sys.exit(f"setup: {role}.yaml still has unrendered placeholders") out = ADAPTERS / f"{role}.yaml" + # A bare `docker compose up -d` before this script runs starts the + # adapters too, and Docker creates a DIRECTORY at a bind-mount source + # that does not exist. Writing would then fail with a bare + # IsADirectoryError that says nothing about the cause. + if out.is_dir(): + sys.exit( + f"setup: {out} is a directory, not a file.\n" + f" Docker created it, which means the adapters were started before this\n" + f" script ran. Bring them down, remove the empty directories and retry:\n" + f" docker compose down\n" + f" rmdir config/adapters/*.yaml\n" + f" docker compose up -d registry discovery && python3 bin/setup.py") out.write_text(template) out.chmod(0o600) # holds a private key print(f" config/adapters/{role}.yaml") diff --git a/docker-deployment/config/registry/schemas/Participant.json b/docker-deployment/config/registry/schemas/Participant.json index ab04c8f..a809ec1 100644 --- a/docker-deployment/config/registry/schemas/Participant.json +++ b/docker-deployment/config/registry/schemas/Participant.json @@ -40,22 +40,22 @@ "$ref": "#/definitions/Status" }, "baseUrl": { - "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https, except that an upstream with auth.scheme 'none' may be plaintext.", + "description": "The base something is appended to: a Beckn action for a node, a binding's path for an upstream. https for a node, since that is its wire identity; an upstream may be plaintext, because nothing in the registry is sent to it.", "type": "string", "maxLength": 2000, "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" }, "role": { - "description": "What this node does on the network. BAP asks. BPP answers. NETWORK exposes publish and discover, answering discover from published catalogs.", + "description": "What this party does on the network. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs.", "type": "string", "enum": [ - "BAP", - "BPP", - "NETWORK" + "consumer", + "provider", + "network" ] }, "keys": { - "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used in the Authorization keyId.", + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header.", "type": "array", "minItems": 1, "maxItems": 8, @@ -63,14 +63,11 @@ "items": { "$ref": "#/definitions/PublicKey" } - }, - "auth": { - "$ref": "#/definitions/Auth" } }, "allOf": [ { - "description": "A node speaks Beckn: it needs a role and keys, has no credential of ours to present, and its id is its wire identity so it must be a hostname.", + "description": "A node speaks Beckn: it needs a role and keys, and its id is its wire identity so it must be a hostname.", "if": { "properties": { "type": { @@ -86,15 +83,6 @@ "role", "keys" ], - "not": { - "anyOf": [ - { - "required": [ - "auth" - ] - } - ] - }, "properties": { "participantId": { "maxLength": 253, @@ -107,7 +95,7 @@ } }, { - "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify, only a credential we present.", + "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify.", "if": { "properties": { "type": { @@ -119,9 +107,6 @@ ] }, "then": { - "required": [ - "auth" - ], "not": { "anyOf": [ { @@ -137,66 +122,22 @@ ] } } - }, - { - "description": "A credential over plaintext http is a leaked credential.", - "if": { - "properties": { - "auth": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "not": { - "const": "none" - } - } - } - } - }, - "required": [ - "auth" - ] - }, - "then": { - "properties": { - "baseUrl": { - "pattern": "^https://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" - } - } - } } ] }, "PublicKey": { - "description": "A public key, held as material: it is public, so there is nothing to protect. Contrast Secret, always a pointer.", + "description": "A public key, held as material: it is public, so there is nothing to protect. Identified by the osid the registry assigns on write -- which is what a sender names in the Authorization header, and so the only id a verifier can look up. alg carries the purpose: ed25519 signs, x25519 encrypts.", "type": "object", "additionalProperties": false, "required": [ - "keyId", - "use", "alg", "key", "validFrom", "status" ], "properties": { - "keyId": { - "description": "Second field of the Authorization keyId, so a sender can say which key it signed with.", - "type": "string", - "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$" - }, - "use": { - "description": "sign verifies signatures; encrypt is for encrypted payloads.", - "type": "string", - "enum": [ - "sign", - "encrypt" - ] - }, "alg": { - "description": "Fixed by use. Both curves are 32-byte keys.", + "description": "The purpose, and the only thing that carries it now: ed25519 signs, x25519 encrypts. Both curves are 32-byte keys.", "type": "string", "enum": [ "ed25519", @@ -204,9 +145,9 @@ ] }, "key": { - "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. A truncated or wrong-curve key fails at write time.", + "description": "base64 of the 32 raw bytes: 44 chars, one trailing '='. Bare -- no encoding label in front of it. A truncated or wrong-curve key fails at write time.", "type": "string", - "pattern": "^base64:[A-Za-z0-9+/]{43}=$" + "pattern": "^[A-Za-z0-9+/]{43}=$" }, "validFrom": { "type": "string", @@ -225,260 +166,7 @@ "revoked" ] } - }, - "allOf": [ - { - "if": { - "properties": { - "use": { - "const": "sign" - } - } - }, - "then": { - "properties": { - "alg": { - "const": "ed25519" - } - } - } - }, - { - "if": { - "properties": { - "use": { - "const": "encrypt" - } - } - }, - "then": { - "properties": { - "alg": { - "const": "x25519" - } - } - } - } - ] - }, - "Auth": { - "description": "How our adapter authenticates TO an upstream. Not Beckn signing \u2014 that uses Node.keys.", - "type": "object", - "additionalProperties": false, - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "type": "string", - "enum": [ - "none", - "apiKeyQuery", - "apiKeyHeader", - "basic" - ] - }, - "paramName": { - "description": "The single query-parameter or header name the credential goes in.", - "$ref": "#/definitions/ParamName" - }, - "valuePrefix": { - "description": "Prepended to the credential, trailing space included \u2014 'Bearer '. Header schemes only.", - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,30} $" - }, - "paramNames": { - "description": "For an upstream wanting several named credentials. Keys must match `secrets` exactly.", - "type": "object", - "minProperties": 1, - "additionalProperties": { - "$ref": "#/definitions/ParamName" - } - }, - "secrets": { - "description": "Pointers to credentials held outside the registry. Redacted from /search by _osConfig.privateFields.", - "type": "object", - "minProperties": 1, - "additionalProperties": { - "$ref": "#/definitions/Secret" - } - } - }, - "allOf": [ - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "const": "none" - } - } - }, - "then": { - "allOf": [ - { - "not": { - "required": [ - "secrets" - ] - } - }, - { - "not": { - "required": [ - "paramName" - ] - } - }, - { - "not": { - "required": [ - "paramNames" - ] - } - }, - { - "not": { - "required": [ - "valuePrefix" - ] - } - } - ] - } - }, - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "enum": [ - "apiKeyQuery", - "apiKeyHeader" - ] - } - } - }, - "then": { - "required": [ - "secrets" - ], - "oneOf": [ - { - "required": [ - "paramName" - ], - "not": { - "required": [ - "paramNames" - ] - }, - "properties": { - "secrets": { - "maxProperties": 1 - } - } - }, - { - "required": [ - "paramNames" - ], - "allOf": [ - { - "not": { - "required": [ - "paramName" - ] - } - }, - { - "not": { - "required": [ - "valuePrefix" - ] - } - } - ] - } - ] - } - }, - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "const": "apiKeyQuery" - } - } - }, - "then": { - "not": { - "required": [ - "valuePrefix" - ] - } - } - }, - { - "if": { - "required": [ - "scheme" - ], - "properties": { - "scheme": { - "const": "basic" - } - } - }, - "then": { - "required": [ - "secrets" - ], - "properties": { - "secrets": { - "required": [ - "username", - "password" - ] - } - }, - "allOf": [ - { - "not": { - "required": [ - "paramName" - ] - } - }, - { - "not": { - "required": [ - "paramNames" - ] - } - }, - { - "not": { - "required": [ - "valuePrefix" - ] - } - } - ] - } - } - ] - }, - "ParamName": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" + } }, "ParticipantId": { "description": "Stable id, and the only id. For a node this is its network identity -- what field 1 of the Authorization header names, and what a signature is verified against; for an upstream it is the Beckn offer.provider.id.", @@ -493,15 +181,6 @@ "inactive" ] }, - "MaterialRef": { - "description": "A pointer to SECRET material held outside the registry, :. Never the material. Public keys are not MaterialRefs: PublicKey.key holds the bytes.", - "type": "string", - "maxLength": 1024, - "pattern": "^(env://[A-Z][A-Z0-9_]{0,63}|inline:[!-~][ -~]{0,998})$" - }, - "Secret": { - "$ref": "#/definitions/MaterialRef" - }, "ParticipantType": { "description": "node speaks Beckn. upstream is an API we call over ordinary HTTP and does not.", "type": "string", @@ -520,9 +199,7 @@ "type", "baseUrl" ], - "privateFields": [ - "$.auth.secrets" - ], + "privateFields": [], "roles": [ "registryOperator" ], diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 2824c4e..cb451b6 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -1,11 +1,19 @@ # The whole OAN stack in one file: registry, discovery, and the three adapters. # Meant for a shared DEV deployment on a VM. # -# 1. cp .env.example .env set the image tags, and change every -# credential -- they are shipped defaults -# 2. docker compose up -d pulls and starts registry and discovery -# 3. python3 bin/setup.py keys, the three adapter entries, configs -# 4. docker compose up -d again, so the adapters read the configs +# 1. cp .env.example .env change every credential -- they are +# shipped defaults +# 2. docker compose up -d registry discovery +# everything EXCEPT the adapters +# 3. python3 bin/setup.py keys, the three adapter entries, and +# the adapter configs they mount +# 4. docker compose up -d now the adapters +# +# Naming registry and discovery in step 2 is not tidiness. An adapter config +# is a bind-mounted FILE, and Docker creates a DIRECTORY at any bind-mount +# source that does not exist yet -- so starting an adapter before step 3 both +# wedges that container on a directory it cannot parse and leaves a directory +# where step 3 needs to write a file. # # Nothing is built here. The adapter and discovery images are pulled from the # tags named in .env. From 21afd1b9686d272b88dc4aed509a74bc8ed59228 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 15:19:32 +0530 Subject: [PATCH 10/81] docs: describe what a binding-key mismatch actually answers [OpenAgriNet/network-adapter#4] The README, .env.example and setup.py all said a mismatch comes back as a bare {"status":"ACK"} with no on_select, looking as though nothing happened. That is wrong, and it sent a reader looking for a silent failure that does not occur. Both cases were run against the stack to see what they really answer: the payload names a provider the adapter is not configured for 404 NET_ENTITY_NOT_FOUND, "this module serves no capability matching the request". The step passes through and nothing behind it answers, so the refusal is explicit and names the cause. the adapter is configured for the key but the registry has no matching ProviderSchema row 502 with an EMPTY body. This is the one worth documenting: nothing in the response says why, and "no call plan for " appears only in the provider adapter's log. Both are now described with the command that surfaces the second, and the consequence of a single configured key is stated where it will be read: while the adapter carries one, onboarding a second provider is an edit to .env, a re-run of setup.py and a restart -- a registry entry alone is not enough. Whether the adapter should instead match on capability, and serve any provider the registry lists for it, is a separate question and deliberately still open. --- docker-deployment/.env.example | 23 +++++++++++++++++------ docker-deployment/README.md | 29 ++++++++++++++++++++++------- docker-deployment/bin/setup.py | 8 +++++--- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index 9d88e54..5c241e3 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -89,12 +89,23 @@ PROVIDER_SUBSCRIBER_ID=provider.oan.dev # against the keys in its own config. setup.py renders these two values into # that config as the one key it answers to. # -# When they disagree the step does not fail. It concludes the request is meant -# for some other provider and passes it through untouched, which is exactly -# what lets one adapter host several capabilities. Nothing further answers, so -# the reply is a bare {"status":"ACK"} with no on_select -- the request looks -# accepted and silently does nothing. These must therefore equal participantId -# and capabilityCode in the ProviderSchema row, exactly. +# A disagreement fails, and how it fails depends on which side is wrong: +# +# the payload names a provider this adapter is not configured for +# 404 NET_ENTITY_NOT_FOUND, "this module serves no capability matching the +# request". The step passes it through -- which is what lets one adapter +# host several capabilities -- and nothing behind it answers. +# +# the adapter IS configured for the key but the registry has no matching +# ProviderSchema row +# 502 with an EMPTY body. The reason, "no call plan for ", appears +# only in the provider adapter's log: +# docker compose logs provider-adapter | grep "no call plan" +# +# So these two must equal participantId and capabilityCode in the +# ProviderSchema row, exactly. While the adapter carries ONE configured key, +# onboarding a second provider is an edit here, a re-run of bin/setup.py and a +# restart -- a registry entry alone is not enough. PROVIDER_PARTICIPANT_ID=my-weather-api PROVIDER_CAPABILITY=openagrinet:WeatherObservation diff --git a/docker-deployment/README.md b/docker-deployment/README.md index 45d9de9..c9cd1cb 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -363,8 +363,9 @@ show up. ## When it does not work -**`{"status":"ACK"}` and no `on_select`.** The commonest one. The provider -adapter did not recognise the request as its own, so it passed it through. +**404 `NET_ENTITY_NOT_FOUND`, "this module serves no capability matching the +request".** The commonest one. The provider adapter did not recognise the +request as its own, so it passed it through and nothing behind it answered. It decides that by building a binding key from the incoming payload — the provider id at `message.contract.commitments[].offer.provider.id` and the @@ -372,11 +373,25 @@ capability at `...resources[].resourceAttributes.@type` — and comparing it against the keys in its own config, which `setup.py` rendered from `PROVIDER_PARTICIPANT_ID|PROVIDER_CAPABILITY`. -A mismatch is not an error, by design: passing through is what lets one adapter -host several capabilities. But nothing further answers, so the request looks -accepted and silently does nothing. Compare all three — the payload, the -`ProviderSchema` row, and `.env` — and re-run `bin/setup.py` after changing -`.env`. +Passing through is deliberate: it is what lets one adapter host several +capabilities. Compare all three — the payload, the `ProviderSchema` row, and +`.env` — and re-run `bin/setup.py` after changing `.env`. + +While the adapter carries one configured key, onboarding a second provider is +an edit to `.env`, a re-run of `bin/setup.py` and a restart of the provider +adapter. A registry entry on its own is not enough. + +**502 with an empty body.** The adapter *is* configured for the key, but the +registry has no matching `ProviderSchema` row, so no call plan resolves. +Nothing in the response says so — the reason is in the log: + +```sh +docker compose logs provider-adapter | grep "no call plan" +``` + +Check the row exists and that its `bindingKey` matches character for +character. The registry is append-only, so a mistyped row cannot be edited — +only superseded under a new id. **The adapters restart in a loop on the first `up`.** Expected before `bin/setup.py` has run — there is no `config/adapters/*.yaml` yet. diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 2a3b323..835c574 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -329,6 +329,8 @@ def render(identities): 3. docker compose up -d The bindingKey above is what the provider adapter was just configured to -answer to. If the ProviderSchema row says anything else, the adapter concludes -the request is not its own and passes it through -- and the reply is a bare -ACK with no on_select, which looks like nothing happened.""") +answer to, and the ProviderSchema row has to match it exactly. A payload +naming anything else is answered 404 "this module serves no capability +matching the request". A row that is missing, while the adapter is configured +for the key, is answered 502 with an empty body and explained only in +`docker compose logs provider-adapter`.""") From 21de2982b9cf741f34cfa1bc191671478e97ddcc Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 15:19:51 +0530 Subject: [PATCH 11/81] docs: fix a cross-reference left pointing at the old symptom [OpenAgriNet/network-adapter#4] The troubleshooting entry it names was renamed in the previous commit, so the pointer described a symptom that no longer appears anywhere. --- docker-deployment/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-deployment/README.md b/docker-deployment/README.md index c9cd1cb..12b9043 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -186,7 +186,8 @@ Things worth knowing about these two calls: it. - **`bindingKey` is `participantId|capabilityCode`.** It has to match what `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` were set to in `.env` - when `setup.py` last ran — see the troubleshooting note on bare ACKs. + when `setup.py` last ran — see the troubleshooting section for what a + mismatch answers. - **`path` must start with one `/` and contain no empty segment.** The schema refuses `//get-daily`, and so does the adapter. - **This registry is append-only.** There is no update, delete is soft, and a From 1285746b2e548fd2cc9f2ee4ea6aef0fcfff3004 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:33:58 +0530 Subject: [PATCH 12/81] chore: add chart conventions, lint script, and helm lint CI [#53] Establishes the repo-level scaffolding the charts depend on: naming and structure conventions, a lint script that rebuilds file:// dependencies before linting, and a CI workflow that runs it. .gitignore excludes Helm dependency artifacts, which are regenerated rather than committed. --- .github/workflows/helm-lint.yml | 32 +++++++++ .gitignore | 15 ++++ CONVENTIONS.md | 123 ++++++++++++++++++++++++++++++++ scripts/lint-charts.sh | 87 ++++++++++++++++++++++ 4 files changed, 257 insertions(+) create mode 100644 .github/workflows/helm-lint.yml create mode 100644 .gitignore create mode 100644 CONVENTIONS.md create mode 100755 scripts/lint-charts.sh diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml new file mode 100644 index 0000000..4e4fabe --- /dev/null +++ b/.github/workflows/helm-lint.yml @@ -0,0 +1,32 @@ +name: helm-lint + +on: + pull_request: + paths: + - 'charts/**' + - 'scripts/lint-charts.sh' + - '.github/workflows/helm-lint.yml' + push: + branches: + - main + - development + paths: + - 'charts/**' + - 'scripts/lint-charts.sh' + - '.github/workflows/helm-lint.yml' + +jobs: + lint: + name: lint and render charts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: v3.16.4 + + - name: Lint and render all charts + run: ./scripts/lint-charts.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5874466 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# macOS +.DS_Store + +# Helm dependency artifacts (regenerated by `helm dependency update`) +charts/**/charts/ +charts/**/Chart.lock +*.tgz + +# Rendered output +rendered/ + +# Editor/IDE +.vscode/ +.idea/ +*.swp diff --git a/CONVENTIONS.md b/CONVENTIONS.md new file mode 100644 index 0000000..daff3d6 --- /dev/null +++ b/CONVENTIONS.md @@ -0,0 +1,123 @@ +# Chart conventions + +Rules every chart in this repository follows. They exist so that a service chart +is predictable to read, safe to upgrade, and traceable back to the change that +produced it. + +## Naming + +| Thing | Rule | Example | +|---|---|---| +| Chart directory and `name` | `oan-`, kebab-case, matching the service's repo/deployment name | `oan-registry-service` | +| Library chart | `oan-common` — the only library chart; every service chart depends on it | `oan-common` | +| Reference chart | `oan-template` — copied to start a new chart, never deployed as-is | `oan-template` | +| A deployed component | Named after the **role it plays in OAN**, no `oan-` prefix — the prefix is for the shared library and the starter chart | `registry`, `keycloak` | +| Two charts for the same role | Add the distinguishing implementation as a suffix, only when there is something to distinguish | `postgresql-cnpg` | +| Release name | The service name without the `oan-` prefix, so resources read `registry-service-...` not `oan-registry-service-oan-registry-service` | `helm install registry-service charts/oan-registry-service` | +| Template helpers | Chart-local helpers are namespaced by chart name: `.` | `oan-registry-service.fullname` | +| Value keys | camelCase, matching Kubernetes field names where one exists | `podSecurityContext`, `envFromSecrets` | +| Env var keys in `envConfig` | SCREAMING_SNAKE_CASE | `LOG_LEVEL` | +| Custom labels/annotations | Prefixed with a domain we own | `oan.in/environment: dev` | + +Component charts are named after the role the component plays in OAN, not after +the software that happens to implement it: `registry` and `keycloak`, not +`registry-sunbird-rc` or `keycloak-sunbird-rc`. The implementation is an +implementation detail, and one that can change without the role changing. + +Add an implementation suffix only when it actually distinguishes something — +`postgresql-cnpg` carries `-cnpg` because a plain `postgresql` chart could +reasonably mean several different operators, and which one is in use changes how +the chart is configured and operated. + +The `oan-` prefix is reserved for the shared library (`oan-common`) and the +starter chart (`oan-template`). Every chart, prefixed or not, depends on +`oan-common` and carries the standard OAN labels, including +`app.kubernetes.io/part-of: oan`. + +Chart directory name, `name` in `Chart.yaml`, and the prefix of the chart-local +helpers must all agree. A mismatch is the most common cause of a chart that +lints clean but renders the wrong resource names. + +## Versioning + +Two independent version fields, both required in every `Chart.yaml`: + +- **`version`** — the version of the *chart*, following + [Semantic Versioning](https://semver.org/). Bumped on every chart change, + even a comment-only one. +- **`appVersion`** — the version of the *application image* the chart deploys by + default, quoted. It is the fallback for `image.tag`, so it must be a real, + pullable tag. Bumping it is a chart change, and so requires a `version` bump + too. + +`version` bump rules: + +| Change | Bump | +|---|---| +| New value key with a backward-compatible default; new optional resource | MINOR | +| Bug fix in a template; doc/comment change; `appVersion` bump | PATCH | +| Removing or renaming a value key or helper; changing a default that alters live behaviour; changing an immutable field such as a selector label | MAJOR | + +For `oan-common` specifically: consumers pin `version: "0.1.x"`, so helper +additions ship as PATCH/MINOR and MAJOR is reserved for renaming or changing the +behaviour of an existing helper. A MAJOR bump of `oan-common` means every +consuming chart's pin has to be updated deliberately. + +Pre-1.0.0 charts are still in flux; once a chart is deployed to production it +goes to `1.0.0` and the rules above are binding. + +## Changelog + +Every chart keeps a `CHANGELOG.md` in +[Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. A chart change +is not complete without both the `version` bump and the matching changelog +entry. This is what makes "which chart version introduced this?" answerable. + +## Required in every service chart + +The deployment epic requires these on every component, and `oan-common` +enforces the first two at render time rather than leaving them to review: + +1. **Resource requests and limits** — `oan-common.resources` fails the render + when `.Values.resources` is empty. This applies to data stores too, where the + requests land on the operator-managed pods. +2. **Liveness and readiness probes** — enabled by default in `oan-template`. + `oan-common.probeSpec` fails the render when an enabled probe declares no + handler, or declares more than one (which the API server would otherwise + reject at apply time, long after the render looked fine). +3. **A ServiceAccount per service** — never the namespace `default`. Attach IRSA + role ARNs via `serviceAccount.annotations`. +4. **Standard labels** — from `oan-common.labels`, giving every resource + `app.kubernetes.io/*` plus `app.kubernetes.io/part-of: oan`. + +## Secrets + +No secret value is ever committed to this repository — not in `values.yaml`, not +in a per-environment values file. + +- **No chart in this repository renders a Secret.** Charts only *reference* + Secrets by name, so the question of how secret material gets into the cluster + is answered once, outside the charts, rather than differently per chart. +- `envFromSecrets` lists names of Secrets whose keys become environment + variables. `secretEnv` maps one Secret key to one variable name, for when the + producing key and the expected variable differ. +- `envConfig` is for non-secret configuration only. It lands in a ConfigMap. + +## Validation + +`./scripts/lint-charts.sh` runs `helm lint --strict` on every chart and +`helm template` on every application chart. CI runs the same script on pull +requests and on pushes to `main` and `development`, so a chart that fails +locally fails the same way in CI. + +Because service charts depend on `oan-common` through +`file://../oan-common`, and the packaged dependency is not committed, an edit to +`oan-common` is only visible to a consuming chart after +`helm dependency update charts/` (or a run of the lint script). + +## Traceability + +Chart work follows the repository's git conventions: branch +`feat/-`, commits carrying `[#]`, PR body closing the issue. +The chart's changelog entry and the issue number are the two ends of the same +thread. diff --git a/scripts/lint-charts.sh b/scripts/lint-charts.sh new file mode 100755 index 0000000..dddf9a2 --- /dev/null +++ b/scripts/lint-charts.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Lint and render every chart under charts/. +# +# Run locally exactly as CI runs it: +# ./scripts/lint-charts.sh +# +# Charts depending on oan-common via file://../oan-common carry no committed +# dependency artifact, so the dependency is rebuilt here before linting. That +# also means a local edit to oan-common is only picked up after this runs (or +# after `helm dependency update `). +# +# A chart whose defaults deliberately fail the render - because it requires a +# database host or a secret reference that has no safe default - supplies the +# minimum needed to template in ci/lint-values.yaml. Every file matching +# ci/*-values.yaml is rendered as a separate case, so a chart can cover several +# configurations. +set -euo pipefail + +cd "$(dirname "$0")/.." + +failed=0 + +for chart_dir in charts/*/; do + chart_dir="${chart_dir%/}" + [[ -f "$chart_dir/Chart.yaml" ]] || continue + + name="$(basename "$chart_dir")" + chart_type="$(helm show chart "$chart_dir" 2>/dev/null | awk '/^type:/ {print $2}')" + chart_type="${chart_type:-application}" + + echo "==> $name (type: $chart_type)" + + if grep -q '^dependencies:' "$chart_dir/Chart.yaml"; then + helm dependency update "$chart_dir" >/dev/null + fi + + # Render once per ci/*-values.yaml, or once with plain defaults if there are none. + shopt -s nullglob + ci_values=("$chart_dir"/ci/*-values.yaml) + shopt -u nullglob + + # Pass the same values to lint. Without them, lint renders bare defaults and + # reports the chart's own "this value is required" failures as [INFO] Fail + # lines - noise that looks like a broken chart but is the guardrail working. + # Written this way for bash 3.2 (macOS), where "${empty_array[@]}" under + # `set -u` is an unbound-variable error rather than an empty expansion. + lint_args=() + if [[ ${#ci_values[@]} -gt 0 ]]; then + for values in "${ci_values[@]}"; do + lint_args+=(-f "$values") + done + fi + + if ! helm lint --strict "$chart_dir" "${lint_args[@]+"${lint_args[@]}"}"; then + echo "!!! helm lint failed for $name" + failed=1 + continue + fi + + # Library charts render no resources of their own, so there is nothing to template. + if [[ "$chart_type" == "library" ]]; then + continue + fi + + if [[ ${#ci_values[@]} -eq 0 ]]; then + if ! helm template "$name" "$chart_dir" >/dev/null; then + echo "!!! helm template failed for $name" + failed=1 + fi + else + for values in "${ci_values[@]}"; do + echo " render: $(basename "$values")" + if ! helm template "$name" "$chart_dir" -f "$values" >/dev/null; then + echo "!!! helm template failed for $name with $(basename "$values")" + failed=1 + fi + done + fi +done + +if [[ "$failed" -ne 0 ]]; then + echo "chart validation FAILED" + exit 1 +fi + +echo "all charts linted and rendered successfully" From b732a5597d7f05b302fbc7711b74f8647809ca6a Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:34:15 +0530 Subject: [PATCH 13/81] feat: add oan-common library chart and oan-template scaffold [#53] oan-common holds the shared helpers every OAN chart renders through - names, labels, image references, resources, and security contexts - so the service charts stay declarative. Resources are deliberately required rather than defaulted: an unset value fails the render instead of shipping an unbounded pod. oan-template is the copy-from starting point for a new service chart. --- charts/oan-common/.helmignore | 10 + charts/oan-common/CHANGELOG.md | 53 +++ charts/oan-common/Chart.yaml | 17 + charts/oan-common/README.md | 117 +++++ charts/oan-common/templates/_helpers.tpl | 402 ++++++++++++++++++ charts/oan-common/values.yaml | 151 +++++++ charts/oan-template/.helmignore | 10 + charts/oan-template/CHANGELOG.md | 37 ++ charts/oan-template/Chart.yaml | 22 + charts/oan-template/README.md | 121 ++++++ charts/oan-template/ci/lint-values.yaml | 9 + charts/oan-template/templates/NOTES.txt | 28 ++ charts/oan-template/templates/_helpers.tpl | 45 ++ charts/oan-template/templates/configmap.yaml | 12 + charts/oan-template/templates/deployment.yaml | 79 ++++ charts/oan-template/templates/ingress.yaml | 41 ++ charts/oan-template/templates/service.yaml | 19 + .../templates/serviceaccount.yaml | 13 + charts/oan-template/values.yaml | 155 +++++++ 19 files changed, 1341 insertions(+) create mode 100644 charts/oan-common/.helmignore create mode 100644 charts/oan-common/CHANGELOG.md create mode 100644 charts/oan-common/Chart.yaml create mode 100644 charts/oan-common/README.md create mode 100644 charts/oan-common/templates/_helpers.tpl create mode 100644 charts/oan-common/values.yaml create mode 100644 charts/oan-template/.helmignore create mode 100644 charts/oan-template/CHANGELOG.md create mode 100644 charts/oan-template/Chart.yaml create mode 100644 charts/oan-template/README.md create mode 100644 charts/oan-template/ci/lint-values.yaml create mode 100644 charts/oan-template/templates/NOTES.txt create mode 100644 charts/oan-template/templates/_helpers.tpl create mode 100644 charts/oan-template/templates/configmap.yaml create mode 100644 charts/oan-template/templates/deployment.yaml create mode 100644 charts/oan-template/templates/ingress.yaml create mode 100644 charts/oan-template/templates/service.yaml create mode 100644 charts/oan-template/templates/serviceaccount.yaml create mode 100644 charts/oan-template/values.yaml diff --git a/charts/oan-common/.helmignore b/charts/oan-common/.helmignore new file mode 100644 index 0000000..7e6c23d --- /dev/null +++ b/charts/oan-common/.helmignore @@ -0,0 +1,10 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig diff --git a/charts/oan-common/CHANGELOG.md b/charts/oan-common/CHANGELOG.md new file mode 100644 index 0000000..50a83af --- /dev/null +++ b/charts/oan-common/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +All notable changes to the `oan-common` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Added +- `oan-common.image` now fails the render when `image.repository` is empty. + Previously an empty repository produced a syntactically valid but meaningless + reference such as `ghcr.io/:v2.0.0`, which Helm and the API server both accept + and which only surfaces later as an `ImagePullBackOff` - long after the deploy + appeared to succeed. + +### Removed +- `oan-common.externalsecret`, `oan-common.externalSecret.enabled` and + `oan-common.externalSecret.targetName`, along with the `externalSecrets` value + block. External Secrets Operator is not installed in any OAN cluster, so no + consuming chart could exercise them. + + The helpers that reference existing Secrets - `oan-common.env` for `secretEnv`, + and each chart's own `*Secret.name` settings - are unchanged. + +## [0.1.0] - 2026-08-31 + +### Added +- Initial release of the `oan-common` library chart. +- Name, fullname, chart, standard label, selector label, annotation, and + namespace helpers. +- Image reference and image pull secret helpers, with digest pinning + (`image.digest`) taking precedence over `image.tag`. +- Service account name/enabled helpers. +- Env ConfigMap name/data helpers and an `envConfig` checksum annotation helper. +- `oan-common.env`, rendering container env entries from `secretEnv` (mapping an + env var name to a specific Secret key) and `extraEnv` (raw passthrough), and + failing the render when a `secretEnv` entry is missing its name or key. +- `oan-common.resources`, which fails the render when `resources` is empty so no + component can ship without a resource contract. +- `oan-common.probes` and `oan-common.probeSpec`, passing every probe field + through verbatim and failing the render when an enabled probe declares no + handler or more than one. +- Pod-level and container-level security context helpers, gated on `enabled`. +- `oan-common.externalsecret`, rendering a complete External Secrets Operator + `ExternalSecret`, with target name and enabled helpers, and render-time + validation of `secretStoreRef.name` and `data`/`dataFrom`. +- `oan-common.waitFor`, rendering init containers that block startup until TCP + and HTTP dependencies are reachable - the missing equivalent of compose's + `depends_on: condition: service_healthy`. Checks are passed in explicitly so a + consuming chart derives host and URL from its own settings rather than + duplicating them. +- Deployment and Ingress apiVersion helpers. diff --git a/charts/oan-common/Chart.yaml b/charts/oan-common/Chart.yaml new file mode 100644 index 0000000..87c04ac --- /dev/null +++ b/charts/oan-common/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: oan-common +description: Common Helm library chart for OpenAgriNet (OAN) services +type: library +version: 0.2.0 +appVersion: "0.1.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - common + - library +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/oan-common/README.md b/charts/oan-common/README.md new file mode 100644 index 0000000..a254a00 --- /dev/null +++ b/charts/oan-common/README.md @@ -0,0 +1,117 @@ +# oan-common + +The common Helm **library chart** for OpenAgriNet (OAN) services. + +`oan-common` renders **no resources of its own** and is never installed +directly. Service charts declare it as a dependency and call its named template +helpers, so names, labels, image references, probes, resource contracts, and +secret wiring are identical across every OAN chart. + +## Using it in a chart + +1. Declare the dependency in your chart's `Chart.yaml`: + + ```yaml + dependencies: + - name: oan-common + version: "0.1.x" + repository: "file://../oan-common" + ``` + +2. Pull it in: + + ```bash + helm dependency update charts/ + ``` + + The packaged dependency is not committed, so re-run this after every edit to + `oan-common` — otherwise your chart keeps rendering against a stale copy. + +3. Define thin chart-local wrappers in your `templates/_helpers.tpl` that + delegate to the library: + + ```yaml + {{- define "my-service.fullname" -}} + {{- include "oan-common.fullname" . -}} + {{- end }} + ``` + + See [`oan-template`](../oan-template) for a complete, copy-ready example. + +## Helpers + +### Names and labels + +| Helper | Purpose | +|---|---| +| `oan-common.name` | Chart name, honoring `nameOverride` | +| `oan-common.fullname` | Fully qualified name (`-`), honoring `fullnameOverride` | +| `oan-common.chart` | `name-version` string for the `helm.sh/chart` label | +| `oan-common.labels` | Standard `app.kubernetes.io/*` labels, `part-of: oan`, plus `commonLabels` | +| `oan-common.selectorLabels` | Pod/Service selector labels (name + instance) | +| `oan-common.annotations` | Renders `commonAnnotations` | +| `oan-common.namespace` | Release namespace | + +### Workload + +| Helper | Purpose | +|---|---| +| `oan-common.image` | Full image ref from `image.registry`/`repository`/`tag`, falling back to `Chart.appVersion` then `latest`. `image.digest` pins by digest and wins over the tag. **Fails the render when `repository` is empty** | +| `oan-common.imagePullSecrets` | Renders the `imagePullSecrets` block from `image.pullSecrets` | +| `oan-common.resources` | Renders `resources`. **Fails the render when empty** — every OAN component must declare a resource contract | +| `oan-common.probes` | Renders every enabled probe block (startup, liveness, readiness) for a container spec | +| `oan-common.probeSpec` | Renders one probe. Takes `(dict "probe" "name" "chart" .Chart.Name)` | +| `oan-common.podSecurityContext` | Pod-level security context, only when `podSecurityContext.enabled` | +| `oan-common.securityContext` | Container-level security context, only when `securityContext.enabled` | + +Probes pass every field except `enabled` through verbatim, so any handler +(`httpGet`, `tcpSocket`, `exec`, `grpc`) and any timing field works. Two +render-time guardrails apply: + +- An enabled probe with **no** handler fails the render. +- An enabled probe with **more than one** handler fails the render. This is the + common trap: Helm merges maps, so overriding a default `httpGet` probe with + `tcpSocket` leaves both in the merged value and the API server rejects it at + apply time. Null out the default you are replacing: + + ```bash + --set livenessProbe.tcpSocket.port=http --set livenessProbe.httpGet=null + ``` + +### Service account + +| Helper | Purpose | +|---|---| +| `oan-common.serviceAccount.name` | Service account name (generated, overridden, or `default` when disabled) | +| `oan-common.serviceAccount.enabled` | Emits `true` when a ServiceAccount should be created | + +### Configuration and secrets + +| Helper | Purpose | +|---|---| +| `oan-common.env` | Container `env` entries from `secretEnv` (env var name -> secret key) and `extraEnv` (raw passthrough) | +| `oan-common.envConfigMapName` | Name of the env ConfigMap (`-env`) | +| `oan-common.envConfigMapData` | Renders `envConfig` into ConfigMap `data` entries | +| `oan-common.checksumAnnotation` | Checksum of `envConfig`, to roll pods when config changes | + +### apiVersions + +`oan-common.deployment.apiVersion` and `oan-common.ingress.apiVersion` keep those +in one place, so a Kubernetes upgrade is a single edit. + +## Value schema + +The helpers read the keys documented in [`values.yaml`](./values.yaml): +`nameOverride`, `fullnameOverride`, `image.*` (including `digest`), `serviceAccount.*`, `envConfig`, +`resources`, `secretEnv`, `extraEnv`, `livenessProbe`/`readinessProbe`/`startupProbe`, +`podSecurityContext`, `securityContext`, `waitFor`, `commonLabels`, and +`commonAnnotations`. A consuming chart inherits this schema and extends it with +its own keys (`replicaCount`, `service`, `ingress`, ...). + +## Versioning + +Consumers pin `version: "0.1.x"`. Ship helper additions as PATCH/MINOR; reserve +MAJOR for renaming or changing the behaviour of an existing helper, since that +forces every consuming chart to update its pin. Every change needs a `version` +bump in `Chart.yaml` and an entry in [`CHANGELOG.md`](./CHANGELOG.md) — see +[`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/oan-common/templates/_helpers.tpl b/charts/oan-common/templates/_helpers.tpl new file mode 100644 index 0000000..6504773 --- /dev/null +++ b/charts/oan-common/templates/_helpers.tpl @@ -0,0 +1,402 @@ +{{/* +# ============================================================================ +# OAN COMMON LIBRARY CHART - SHARED HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: reusable template helpers consumed by every OAN service chart. +# This chart renders no resources of its own. +# GitHub: https://github.com/OpenAgriNet/helmcharts/blob/main/charts/oan-common/templates/_helpers.tpl +# ============================================================================ +*/}} + +{{/* +Chart name, honoring nameOverride. +Usage: {{ include "oan-common.name" . }} +*/}} +{{- define "oan-common.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified app name (-), honoring fullnameOverride. +Truncated to 63 chars for the DNS label limit. +Usage: {{ include "oan-common.fullname" . }} +*/}} +{{- define "oan-common.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart name and version for the helm.sh/chart label. +Usage: {{ include "oan-common.chart" . }} +*/}} +{{- define "oan-common.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Standard labels for every resource. +Usage: {{ include "oan-common.labels" . | nindent 4 }} +*/}} +{{- define "oan-common.labels" -}} +helm.sh/chart: {{ include "oan-common.chart" . }} +{{ include "oan-common.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: oan +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels for pods and services. These are immutable on a Deployment +selector, so nothing environment-specific belongs here. +Usage: {{ include "oan-common.selectorLabels" . | nindent 4 }} +*/}} +{{- define "oan-common.selectorLabels" -}} +app.kubernetes.io/name: {{ include "oan-common.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common annotations for every resource. +Usage: {{ include "oan-common.annotations" . | nindent 4 }} +*/}} +{{- define "oan-common.annotations" -}} +{{- with .Values.commonAnnotations }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Name of the service account to use. +Usage: {{ include "oan-common.serviceAccount.name" . }} +*/}} +{{- define "oan-common.serviceAccount.name" -}} +{{- if .Values.serviceAccount.enabled }} +{{- default (include "oan-common.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Emits "true" when a ServiceAccount should be created. +Usage: {{ if include "oan-common.serviceAccount.enabled" . }} +*/}} +{{- define "oan-common.serviceAccount.enabled" -}} +{{- if .Values.serviceAccount.enabled }} +{{- true }} +{{- end }} +{{- end }} + +{{/* +Full image reference: /:, or +/@ when image.digest is set. + +A digest wins over a tag, so an environment can pin an exact image without +having to blank out the tag. + +Tag falls back to Chart.appVersion, then "latest". +Usage: {{ include "oan-common.image" . }} +*/}} +{{- define "oan-common.image" -}} +{{- $registry := .Values.image.registry | default "" }} +{{- $repository := .Values.image.repository | default "" }} +{{- if not $repository }} +{{/* +Without this, an empty repository renders a syntactically valid but meaningless +reference - "ghcr.io/:v2.0.0" - which Helm and the API server both accept. The +failure only surfaces later as an ImagePullBackOff, long after the deploy looked +successful. Fail here instead. +*/}} +{{- fail (printf "%s: image.repository is required - set image.registry/repository/tag for this environment" .Chart.Name) }} +{{- end }} +{{- if $registry }} +{{- $repository = printf "%s/%s" $registry $repository }} +{{- end }} +{{- with .Values.image.digest }} +{{- printf "%s@%s" $repository . }} +{{- else }} +{{- printf "%s:%s" $repository (.Values.image.tag | default $.Chart.AppVersion | default "latest") }} +{{- end }} +{{- end }} + +{{/* +imagePullSecrets block, rendered only when image.pullSecrets is non-empty. +Usage: {{ include "oan-common.imagePullSecrets" . | nindent 6 }} +*/}} +{{- define "oan-common.imagePullSecrets" -}} +{{- with .Values.image.pullSecrets }} +imagePullSecrets: +{{- range . }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Name of the env ConfigMap (-env). +Usage: {{ include "oan-common.envConfigMapName" . }} +*/}} +{{- define "oan-common.envConfigMapName" -}} +{{- printf "%s-env" (include "oan-common.fullname" .) }} +{{- end }} + +{{/* +envConfig rendered as ConfigMap data entries. +Usage: {{ include "oan-common.envConfigMapData" . | nindent 2 }} +*/}} +{{- define "oan-common.envConfigMapData" -}} +{{- range $key, $value := .Values.envConfig }} +{{ $key }}: {{ $value | quote }} +{{- end }} +{{- end }} + +{{/* +Checksum of envConfig, so a config change rolls the pods. +Usage: checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} +*/}} +{{- define "oan-common.checksumAnnotation" -}} +{{- $envConfig := .Values.envConfig | default dict }} +{{- $envConfig | toJson | sha256sum }} +{{- end }} + +{{/* +Container env entries built from `secretEnv` and `extraEnv`. + +`secretEnv` maps an environment variable name to a secret key, for the common +case where the producing secret's key name differs from the variable the app +expects (CNPG writes `password`; Sunbird RC wants `connectionInfo_password`): + + secretEnv: + connectionInfo_password: + name: registry-db-app + key: password + optional: false # optional + +`extraEnv` is a raw list of env entries, passed through verbatim for anything +this schema does not cover (fieldRef, resourceFieldRef, plain values). + +Usage: + {{- with (include "oan-common.env" . | trim) }} + env: + {{- . | nindent 12 }} + {{- end }} +*/}} +{{- define "oan-common.env" -}} +{{- range $name, $ref := .Values.secretEnv }} +{{- if not $ref.name }} +{{- fail (printf "secretEnv.%s.name is required - it must name the Secret holding the value" $name) }} +{{- end }} +{{- if not $ref.key }} +{{- fail (printf "secretEnv.%s.key is required - it must name the key inside Secret %q" $name $ref.name) }} +{{- end }} +- name: {{ $name }} + valueFrom: + secretKeyRef: + name: {{ $ref.name }} + key: {{ $ref.key }} + {{- if hasKey $ref "optional" }} + optional: {{ $ref.optional }} + {{- end }} +{{- end }} +{{- with .Values.extraEnv }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Init containers that block startup until dependencies are reachable. + +Kubernetes has no equivalent of compose's `depends_on: condition: +service_healthy`. Without this a pod starts before its database or its identity +provider is up, fails, and crashloops with backoff - which recovers on its own +but makes a first install look broken and slows every restart. + +Takes the checks explicitly, so a consuming chart derives them from its own +settings (database.host, keycloak.url) instead of making the operator retype +values that would then be free to drift: + + {{- include "oan-common.waitFor" (dict "ctx" . "tcp" $tcp "http" $http) }} + +where $tcp entries are {name, host, port} and $http entries are {name, url}. + +TCP checks use `nc -z`; HTTP checks use `wget --spider`. Both loop until success +or waitFor.timeoutSeconds, then fail the pod so the reason is visible in +`kubectl describe` rather than buried in a crashloop. +*/}} +{{- define "oan-common.waitFor" -}} +{{- $ctx := .ctx -}} +{{- $w := $ctx.Values.waitFor | default dict -}} +{{- if $w.enabled -}} +{{- $img := $w.image | default dict -}} +{{- $image := printf "%s/%s:%s" ($img.registry | default "docker.io") ($img.repository | default "busybox") ($img.tag | default "latest") -}} +{{- $timeout := $w.timeoutSeconds | default 300 -}} +{{- $interval := $w.intervalSeconds | default 3 -}} +{{- $res := $w.resources | default dict -}} +{{- range .tcp }} +{{- if not .host }} +{{- fail (printf "waitFor: the %q check has no host. Set the setting it derives from (e.g. database.host), or disable the check." (.name | default "")) }} +{{- end }} +- name: {{ printf "wait-%s" (.name | default "tcp") | trunc 63 | trimSuffix "-" }} + image: {{ $image | quote }} + imagePullPolicy: {{ $img.pullPolicy | default "IfNotPresent" }} + command: + - /bin/sh + - -c + - | + deadline=$(( $(date +%s) + {{ $timeout }} )) + until nc -z {{ .host | quote }} {{ .port }}; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "timed out after {{ $timeout }}s waiting for {{ .name }} at {{ .host }}:{{ .port }}" + exit 1 + fi + echo "waiting for {{ .name }} at {{ .host }}:{{ .port }}" + sleep {{ $interval }} + done + echo "{{ .name }} is reachable" + resources: + {{- toYaml $res | nindent 4 }} +{{- end }} +{{- range .http }} +{{- if not .url }} +{{- fail (printf "waitFor: the %q check has no url. Set the setting it derives from (e.g. keycloak.url), or disable the check." (.name | default "")) }} +{{- end }} +- name: {{ printf "wait-%s" (.name | default "http") | trunc 63 | trimSuffix "-" }} + image: {{ $image | quote }} + imagePullPolicy: {{ $img.pullPolicy | default "IfNotPresent" }} + command: + - /bin/sh + - -c + - | + deadline=$(( $(date +%s) + {{ $timeout }} )) + until wget -q --spider --timeout=5 {{ .url | quote }}; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "timed out after {{ $timeout }}s waiting for {{ .name }} at {{ .url }}" + exit 1 + fi + echo "waiting for {{ .name }} at {{ .url }}" + sleep {{ $interval }} + done + echo "{{ .name }} is reachable" + resources: + {{- toYaml $res | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Resources block. +Fails the render when resources is empty: every OAN component must declare a +resource contract so the scheduler and the cluster autoscaler have real numbers +to work with. +Usage: {{ include "oan-common.resources" . | nindent 10 }} +*/}} +{{- define "oan-common.resources" -}} +{{- if not .Values.resources -}} +{{- fail (printf "%s: .Values.resources is required - every OAN component must declare requests and limits (see CONVENTIONS.md)" .Chart.Name) -}} +{{- end -}} +{{- toYaml .Values.resources }} +{{- end }} + +{{/* +Render one probe. Everything except the `enabled` flag is passed through +verbatim, so any handler (httpGet, tcpSocket, exec, grpc) and any timing field +works. + +Guardrail: Helm merges maps, so overriding a chart's default httpGet probe with +tcpSocket would leave BOTH handlers in the merged value - which the API server +rejects at apply time, long after the render looked fine. This fails the render +instead and tells you to null out the default. + +Usage: {{ include "oan-common.probeSpec" + (dict "probe" .Values.livenessProbe "name" "livenessProbe" "chart" .Chart.Name) | nindent 2 }} +*/}} +{{- define "oan-common.probeSpec" -}} +{{- $probe := .probe -}} +{{- $name := .name | default "probe" -}} +{{- $chart := .chart | default "chart" -}} +{{- $handlers := list -}} +{{- range $h := list "httpGet" "tcpSocket" "exec" "grpc" -}} +{{- if index $probe $h -}} +{{- $handlers = append $handlers $h -}} +{{- end -}} +{{- end -}} +{{- if gt (len $handlers) 1 -}} +{{- fail (printf "%s: %s declares %d handlers (%s) but a probe may declare only one. Null out the one you do not want, e.g. --set %s.httpGet=null" $chart $name (len $handlers) (join ", " $handlers) $name) -}} +{{- end -}} +{{- if eq (len $handlers) 0 -}} +{{- fail (printf "%s: %s is enabled but declares no handler. Set one of httpGet, tcpSocket, exec or grpc." $chart $name) -}} +{{- end -}} +{{- toYaml (omit $probe "enabled") }} +{{- end }} + +{{/* +All enabled probe blocks (startup, liveness, readiness) for a container spec. +Usage: {{ include "oan-common.probes" . | nindent 8 }} +*/}} +{{- define "oan-common.probes" -}} +{{- if and .Values.startupProbe .Values.startupProbe.enabled }} +startupProbe: + {{- include "oan-common.probeSpec" (dict "probe" .Values.startupProbe "name" "startupProbe" "chart" .Chart.Name) | nindent 2 }} +{{- end }} +{{- if and .Values.livenessProbe .Values.livenessProbe.enabled }} +livenessProbe: + {{- include "oan-common.probeSpec" (dict "probe" .Values.livenessProbe "name" "livenessProbe" "chart" .Chart.Name) | nindent 2 }} +{{- end }} +{{- if and .Values.readinessProbe .Values.readinessProbe.enabled }} +readinessProbe: + {{- include "oan-common.probeSpec" (dict "probe" .Values.readinessProbe "name" "readinessProbe" "chart" .Chart.Name) | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Pod-level security context, rendered only when enabled. +Usage: {{ include "oan-common.podSecurityContext" . | nindent 8 }} +*/}} +{{- define "oan-common.podSecurityContext" -}} +{{- if and .Values.podSecurityContext .Values.podSecurityContext.enabled }} +{{- toYaml (omit .Values.podSecurityContext "enabled") }} +{{- end }} +{{- end }} + +{{/* +Container-level security context, rendered only when enabled. +Usage: {{ include "oan-common.securityContext" . | nindent 10 }} +*/}} +{{- define "oan-common.securityContext" -}} +{{- if and .Values.securityContext .Values.securityContext.enabled }} +{{- toYaml (omit .Values.securityContext "enabled") }} +{{- end }} +{{- end }} + +{{/* +Release namespace. +Usage: {{ include "oan-common.namespace" . }} +*/}} +{{- define "oan-common.namespace" -}} +{{- .Release.Namespace }} +{{- end }} + +{{/* +apiVersion helpers, so a bump lands in one place. +*/}} +{{- define "oan-common.deployment.apiVersion" -}} +apps/v1 +{{- end }} + +{{- define "oan-common.ingress.apiVersion" -}} +networking.k8s.io/v1 +{{- end }} diff --git a/charts/oan-common/values.yaml b/charts/oan-common/values.yaml new file mode 100644 index 0000000..f1517ee --- /dev/null +++ b/charts/oan-common/values.yaml @@ -0,0 +1,151 @@ +# ============================================================================ +# Default values for the oan-common library chart. +# +# A library chart renders no resources of its own. These keys define the value +# schema that consuming charts inherit, and are what the helpers in +# templates/_helpers.tpl read. Consuming charts may extend this schema with +# their own keys (service, ingress, replicaCount, ...). +# ============================================================================ + +# Override the chart name used in names and labels +nameOverride: "" +# Override the generated fullname (default: -) +fullnameOverride: "" + +# Image configuration. Consumed by `oan-common.image`. +image: + # Container registry, e.g. "ghcr.io" or ".dkr.ecr.ap-south-1.amazonaws.com" + registry: "" + # Image repository, e.g. "openagrinet/registry-service" + repository: "" + # Image tag. Falls back to Chart.appVersion, then "latest". + tag: "" + # Image digest (sha256:...). Takes precedence over tag when set, so an + # environment can pin an exact image without blanking the tag. + digest: "" + pullPolicy: IfNotPresent + # Names of image pull secrets in the release namespace + pullSecrets: [] + +# Service account configuration. Consumed by `oan-common.serviceAccount.*`. +serviceAccount: + # Create a dedicated ServiceAccount for this release + enabled: true + # Explicit name. Defaults to the chart fullname when empty. + name: "" + # Annotations, e.g. eks.amazonaws.com/role-arn for IRSA + annotations: {} + automountServiceAccountToken: true + +# Non-secret environment configuration. Rendered into a ConfigMap and exposed +# to the container with envFrom. NEVER put secrets here. +envConfig: {} + +# Resource requests and limits. +# REQUIRED on every OAN component: `oan-common.resources` fails the render when +# this is empty, so a chart cannot ship without a resource contract. +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# Probes. REQUIRED on every OAN component that serves traffic. +# Every field other than `enabled` is passed through to the probe verbatim, so +# any handler (httpGet, tcpSocket, exec, grpc) and any timing field is allowed. +livenessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +startupProbe: + enabled: false + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + +# Pod-level security context. Every field other than `enabled` is passed +# through verbatim. +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +# Container-level security context. Every field other than `enabled` is passed +# through verbatim. +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + +# Init containers that block startup until dependencies are reachable, standing +# in for compose's `depends_on: condition: service_healthy`, which Kubernetes has +# no equivalent of. Consumed by `oan-common.waitFor`. +waitFor: + enabled: false + image: + registry: docker.io + # Needs `nc`, `wget` and `date` - busybox has all three. + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + # Give up after this long, so a genuinely missing dependency surfaces as a + # failed init container rather than an endless wait. + timeoutSeconds: 300 + intervalSeconds: 3 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Which dependencies to wait for is decided by the consuming chart, which + # derives host and URL from its own settings. See that chart's values. + +# Individual environment variables sourced from Secret keys, for when the +# secret's key name differs from the variable the application expects +# (CNPG writes `password`; Sunbird RC wants `connectionInfo_password`). +# Consumed by `oan-common.env`. +secretEnv: {} + # connectionInfo_password: + # name: registry-db-app + # key: password + +# Raw env entries, passed through verbatim (fieldRef, plain values, ...). +extraEnv: [] + +# Extra labels added to every resource +commonLabels: {} +# Extra annotations added to every resource +commonAnnotations: {} diff --git a/charts/oan-template/.helmignore b/charts/oan-template/.helmignore new file mode 100644 index 0000000..7e6c23d --- /dev/null +++ b/charts/oan-template/.helmignore @@ -0,0 +1,10 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig diff --git a/charts/oan-template/CHANGELOG.md b/charts/oan-template/CHANGELOG.md new file mode 100644 index 0000000..2dfa74c --- /dev/null +++ b/charts/oan-template/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +All notable changes to the `oan-template` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +### Added +- Initial release of the `oan-template` reference/starter application chart. +- Depends on the `oan-common` library chart via `file://../oan-common`. +- Templates for Deployment, Service, ServiceAccount, env ConfigMap, optional + ESO ExternalSecret, and optional Ingress, all wired through `oan-common` + helpers. +- Liveness and readiness probes and resource requests/limits enabled by default, + per the deployment epic's requirement for every component. +- Pod-level and container-level security contexts, off by default and enabled + per environment. +- Scheduling controls (`nodeSelector`, `tolerations`, `affinity`) and extra + `podLabels` / `podAnnotations`. +- `envFromSecrets` for pre-existing Secrets not managed by ESO, plus `secretEnv` + and `extraEnv` for individual environment variables. +- Fully commented `values.yaml` and a `NOTES.txt` summarising what was rendered. diff --git a/charts/oan-template/Chart.yaml b/charts/oan-template/Chart.yaml new file mode 100644 index 0000000..6ec645e --- /dev/null +++ b/charts/oan-template/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: oan-template +description: Reference starter chart for OpenAgriNet (OAN) services, built on oan-common +type: application +version: 0.2.0 +appVersion: "1.0.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - template + - starter + - reference +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/oan-template/README.md b/charts/oan-template/README.md new file mode 100644 index 0000000..c85a25d --- /dev/null +++ b/charts/oan-template/README.md @@ -0,0 +1,121 @@ +# oan-template + +The **reference / starter** Helm chart for OpenAgriNet (OAN) services. + +`oan-template` is a complete, working example of an OAN service chart built on +the [`oan-common`](../oan-common) library chart. It is meant to be **copied and +adapted** — not installed as-is. + +## What it renders + +| Resource | File | Notes | +|---|---|---| +| Deployment | `templates/deployment.yaml` | Probes and resources always present; security contexts gated on `enabled` | +| Service | `templates/service.yaml` | `ClusterIP` by default | +| ServiceAccount | `templates/serviceaccount.yaml` | Created when `serviceAccount.enabled` | +| ConfigMap | `templates/configmap.yaml` | Built from `envConfig`; checksum rolls pods on change | +| Ingress | `templates/ingress.yaml` | Optional, off by default | + +Names, labels, image refs, probes, resources, and secret wiring all come from +`oan-common` helpers, so every chart derived from this template stays consistent. + +## How it depends on oan-common + +`Chart.yaml` declares: + +```yaml +dependencies: + - name: oan-common + version: "0.1.x" + repository: "file://../oan-common" +``` + +The chart-local helpers in `templates/_helpers.tpl` are thin wrappers that +delegate to the library (`oan-template.fullname` → `oan-common.fullname`). + +## Create your own chart from this template + +1. Copy the directory and rename it: + + ```bash + cp -r charts/oan-template charts/oan-my-service + ``` + +2. In `charts/oan-my-service/Chart.yaml`, set `name: oan-my-service` and + `appVersion` to the image tag you deploy by default. Keep the `oan-common` + dependency. + +3. Rename the chart-local helpers. Change only the **left side** of each + `define` in `templates/_helpers.tpl` — the `oan-common.*` include inside the + body stays, since that is the shared library you delegate to. Then update the + matching `include "oan-template..."` calls in the template YAML. Both at once, + from the repo root: + + ```bash + grep -rl 'oan-template\.' charts/oan-my-service | xargs sed -i '' 's/oan-template\./oan-my-service./g' + ``` + + (`sed -i ''` is the macOS form; on Linux use `sed -i`.) + +4. Set the real values in `values.yaml`: + - `image.registry` / `image.repository` / `image.tag` + - `service.port` / `service.targetPort` — your container's listen port + - `livenessProbe.httpGet.path` / `readinessProbe.httpGet.path` — your health endpoint + - `resources` — right-sized for the workload + - `envConfig` — non-secret configuration only + +5. Validate: + + ```bash + ./scripts/lint-charts.sh + helm template oan-my-service charts/oan-my-service + ``` + +## Try the template directly + +```bash +helm dependency update charts/oan-template +helm template demo charts/oan-template \ + --set image.repository=nginx --set image.tag=1.27 +``` + +## Configuration + +See [`values.yaml`](./values.yaml) for the full, commented schema. + +Two behaviours worth knowing before you override: + +- **`resources` is mandatory.** Emptying it fails the render by design. +- **Switching a probe handler needs the default nulled out.** Helm merges maps, + so replacing the default `httpGet` probe with `tcpSocket` leaves both handlers + in the merged value. The render fails with an explanatory error; null the one + you are replacing: + + ```bash + --set livenessProbe.tcpSocket.port=http --set livenessProbe.httpGet=null + ``` + +## Secrets + +Never put secret values in `envConfig` or in any committed values file. This +chart renders no Secrets at all — it only *references* them, so they are created +by whatever manages secrets in that environment. + +Two ways to consume one: + +```yaml +# whole Secret injected as env vars +envFromSecrets: + - my-service-db + +# a single key mapped to a specific variable name, for when they differ +secretEnv: + DB_PASSWORD: + name: my-service-db + key: password +``` + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/oan-template/ci/lint-values.yaml b/charts/oan-template/ci/lint-values.yaml new file mode 100644 index 0000000..17cad2c --- /dev/null +++ b/charts/oan-template/ci/lint-values.yaml @@ -0,0 +1,9 @@ +# Minimum values that let `helm template` run in CI. +# +# This chart's image.repository is intentionally empty - it is a starter chart, +# and a chart with no image is not a deployable thing. The render fails without +# one, so CI supplies a placeholder. +image: + registry: docker.io + repository: nginx + tag: "1.27" diff --git a/charts/oan-template/templates/NOTES.txt b/charts/oan-template/templates/NOTES.txt new file mode 100644 index 0000000..1e8e8be --- /dev/null +++ b/charts/oan-template/templates/NOTES.txt @@ -0,0 +1,28 @@ +Thank you for installing {{ .Chart.Name }} (release: {{ .Release.Name }}). + +Rendered from the oan-template reference chart. + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "oan-template.fullname" . }} ({{ .Values.replicaCount }} replica(s)) + - Service/{{ include "oan-template.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "oan-template.envConfigMapName" . }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "oan-template.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "oan-template.fullname" . }} +{{- end }} + +Image in use: + {{ include "oan-template.image" . }} +{{- if not .Values.image.repository }} + +WARNING: image.repository is empty. Set image.registry/repository/tag before a +real deployment, e.g. --set image.repository=openagrinet/my-service +{{- end }} + +Check the rollout: + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "oan-template.fullname" . }} + +Reach the service in-cluster: + {{ include "oan-template.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} diff --git a/charts/oan-template/templates/_helpers.tpl b/charts/oan-template/templates/_helpers.tpl new file mode 100644 index 0000000..29ec803 --- /dev/null +++ b/charts/oan-template/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* +# ============================================================================ +# OAN TEMPLATE CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers that delegate to oan-common. +# +# When you copy this chart, rename every "oan-template." define below to your +# service name and update the matching include calls in the template YAML. +# Change only the LEFT side of each define - the oan-common include inside the +# body is the shared library you delegate to. +# ============================================================================ +*/}} + +{{- define "oan-template.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "oan-template.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "oan-template.chart" -}} +{{- include "oan-common.chart" . -}} +{{- end }} + +{{- define "oan-template.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "oan-template.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "oan-template.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "oan-template.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "oan-template.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + diff --git a/charts/oan-template/templates/configmap.yaml b/charts/oan-template/templates/configmap.yaml new file mode 100644 index 0000000..656ef85 --- /dev/null +++ b/charts/oan-template/templates/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "oan-template.envConfigMapName" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "oan-common.envConfigMapData" . | trim | nindent 2 }} diff --git a/charts/oan-template/templates/deployment.yaml b/charts/oan-template/templates/deployment.yaml new file mode 100644 index 0000000..5de7fa3 --- /dev/null +++ b/charts/oan-template/templates/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "oan-template.fullname" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "oan-template.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "oan-template.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "oan-template.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "oan-template.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + {{- with (include "oan-common.env" . | trim) }} + env: + {{- . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ include "oan-template.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/oan-template/templates/ingress.yaml b/charts/oan-template/templates/ingress.yaml new file mode 100644 index 0000000..964e961 --- /dev/null +++ b/charts/oan-template/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "oan-template.fullname" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "oan-template.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/oan-template/templates/service.yaml b/charts/oan-template/templates/service.yaml new file mode 100644 index 0000000..109557c --- /dev/null +++ b/charts/oan-template/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "oan-template.fullname" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "oan-template.selectorLabels" . | nindent 4 }} diff --git a/charts/oan-template/templates/serviceaccount.yaml b/charts/oan-template/templates/serviceaccount.yaml new file mode 100644 index 0000000..ca5974c --- /dev/null +++ b/charts/oan-template/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "oan-template.serviceAccountName" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/oan-template/values.yaml b/charts/oan-template/values.yaml new file mode 100644 index 0000000..8a06373 --- /dev/null +++ b/charts/oan-template/values.yaml @@ -0,0 +1,155 @@ +# ============================================================================ +# oan-template - default values +# +# This is a REFERENCE / STARTER chart. Copy the whole `oan-template` directory, +# rename it to your service, then adjust the values below. It renders a standard +# stateless service: Deployment + Service + ServiceAccount + env ConfigMap + +# optional Ingress, all wired through oan-common. +# +# Minimum you must change for a real deployment: +# - image.registry / image.repository / image.tag +# - service.port / service.targetPort (your container's listen port) +# - livenessProbe / readinessProbe httpGet.path (your health endpoint) +# - resources (right-size for the workload) +# - envConfig (non-secret configuration only) +# ============================================================================ + +replicaCount: 1 + +# Image configuration. Override registry/repository/tag per environment. +image: + # e.g. "ghcr.io" or ".dkr.ecr.ap-south-1.amazonaws.com" + registry: "" + # e.g. "openagrinet/my-service" + repository: "" + # Leave empty to fall back to Chart.appVersion + tag: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + # For IRSA on EKS: + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + annotations: {} + automountServiceAccountToken: true + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + annotations: {} + +# Ingress. Disabled by default; enable per environment. +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: my-service.local + paths: + - path: / + pathType: Prefix + tls: [] + +# Resource requests and limits. REQUIRED - the render fails if this is empty. +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# Probes. Point httpGet.path at your service's health endpoint. +# Any handler (httpGet, tcpSocket, exec, grpc) and any timing field is passed +# through verbatim; only `enabled` is stripped. +livenessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# Enable for slow-starting services so liveness does not kill them mid-boot. +startupProbe: + enabled: false + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + +# Pod-level security context. Off by default - enable once the image is +# verified to run as a non-root user. +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +# Container-level security context. Off by default; turn on +# readOnlyRootFilesystem once the app writes only to mounted volumes. +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + +# Scheduling +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# ============================================================================ +# ENVIRONMENT CONFIGURATION +# Rendered into a ConfigMap and exposed to the container via envFrom. Pods +# restart automatically when these change (checksum annotation). +# +# NEVER put secrets here. Reference Secrets via envFromSecrets or secretEnv. +# ============================================================================ +envConfig: + LOG_LEVEL: "info" + SERVICE_NAME: "oan-template" + +# Names of pre-existing Secrets in this namespace to inject via envFrom, for +# secrets not managed by ESO. +envFromSecrets: [] + +# Individual env vars sourced from specific Secret keys. Use this when the +# secret's key name differs from the env var the application expects. +secretEnv: {} + # DB_PASSWORD: + # name: my-service-db + # key: password + +# Raw env entries, passed through verbatim. +extraEnv: [] From f1038d305fdbbb404539f303868e0926c35b5ece Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:34:35 +0530 Subject: [PATCH 14/81] feat: add postgresql-cnpg and postgresql-migration charts [#53] postgresql-cnpg renders one CloudNativePG Cluster per release, with optional ScheduledBackup and Barman Cloud ObjectStore. Neither chart renders a password: every credential is a reference to a Secret, and the render fails when one is unset rather than defaulting. Applications connect as the owner of their own database, never as postgres. Extensions that require superuser are created once through bootstrap.postInitApplicationSQL, which the operator runs during bootstrap, so no long-lived role holds the privilege. postgresql-migration runs Flyway as a Job across the databases the cluster chart created. Both target directories carry no SQL yet - Sunbird RC and Keycloak each manage their own schema - so the Job currently skips them and the targets exist as a versioned home. --- charts/postgresql-cnpg/.helmignore | 13 + charts/postgresql-cnpg/CHANGELOG.md | 125 ++++++++ charts/postgresql-cnpg/Chart.yaml | 31 ++ charts/postgresql-cnpg/README.md | 220 ++++++++++++++ charts/postgresql-cnpg/ci/lint-values.yaml | 10 + .../examples/discovery-db.dev.yaml | 139 +++++++++ .../examples/registry-db.dev.yaml | 100 +++++++ charts/postgresql-cnpg/templates/NOTES.txt | 41 +++ charts/postgresql-cnpg/templates/_helpers.tpl | 110 +++++++ charts/postgresql-cnpg/templates/cluster.yaml | 202 +++++++++++++ .../postgresql-cnpg/templates/database.yaml | 64 +++++ .../templates/objectstore.yaml | 41 +++ .../templates/scheduledbackup.yaml | 27 ++ charts/postgresql-cnpg/values.yaml | 271 ++++++++++++++++++ charts/postgresql-migration/.helmignore | 13 + charts/postgresql-migration/CHANGELOG.md | 104 +++++++ charts/postgresql-migration/Chart.yaml | 27 ++ charts/postgresql-migration/README.md | 160 +++++++++++ .../postgresql-migration/ci/lint-values.yaml | 7 + .../ci/no-hook-values.yaml | 11 + .../examples/registry-stack.dev.yaml | 55 ++++ .../files/migrations/01-registry/README.md | 13 + .../files/migrations/02-keycloak/README.md | 10 + .../postgresql-migration/templates/NOTES.txt | 41 +++ .../templates/_helpers.tpl | 146 ++++++++++ .../templates/configmap-migrations.yaml | 29 ++ .../templates/configmap-script.yaml | 87 ++++++ .../templates/configmap.yaml | 25 ++ .../postgresql-migration/templates/job.yaml | 105 +++++++ .../templates/serviceaccount.yaml | 17 ++ charts/postgresql-migration/values.yaml | 194 +++++++++++++ 31 files changed, 2438 insertions(+) create mode 100644 charts/postgresql-cnpg/.helmignore create mode 100644 charts/postgresql-cnpg/CHANGELOG.md create mode 100644 charts/postgresql-cnpg/Chart.yaml create mode 100644 charts/postgresql-cnpg/README.md create mode 100644 charts/postgresql-cnpg/ci/lint-values.yaml create mode 100644 charts/postgresql-cnpg/examples/discovery-db.dev.yaml create mode 100644 charts/postgresql-cnpg/examples/registry-db.dev.yaml create mode 100644 charts/postgresql-cnpg/templates/NOTES.txt create mode 100644 charts/postgresql-cnpg/templates/_helpers.tpl create mode 100644 charts/postgresql-cnpg/templates/cluster.yaml create mode 100644 charts/postgresql-cnpg/templates/database.yaml create mode 100644 charts/postgresql-cnpg/templates/objectstore.yaml create mode 100644 charts/postgresql-cnpg/templates/scheduledbackup.yaml create mode 100644 charts/postgresql-cnpg/values.yaml create mode 100644 charts/postgresql-migration/.helmignore create mode 100644 charts/postgresql-migration/CHANGELOG.md create mode 100644 charts/postgresql-migration/Chart.yaml create mode 100644 charts/postgresql-migration/README.md create mode 100644 charts/postgresql-migration/ci/lint-values.yaml create mode 100644 charts/postgresql-migration/ci/no-hook-values.yaml create mode 100644 charts/postgresql-migration/examples/registry-stack.dev.yaml create mode 100644 charts/postgresql-migration/files/migrations/01-registry/README.md create mode 100644 charts/postgresql-migration/files/migrations/02-keycloak/README.md create mode 100644 charts/postgresql-migration/templates/NOTES.txt create mode 100644 charts/postgresql-migration/templates/_helpers.tpl create mode 100644 charts/postgresql-migration/templates/configmap-migrations.yaml create mode 100644 charts/postgresql-migration/templates/configmap-script.yaml create mode 100644 charts/postgresql-migration/templates/configmap.yaml create mode 100644 charts/postgresql-migration/templates/job.yaml create mode 100644 charts/postgresql-migration/templates/serviceaccount.yaml create mode 100644 charts/postgresql-migration/values.yaml diff --git a/charts/postgresql-cnpg/.helmignore b/charts/postgresql-cnpg/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/postgresql-cnpg/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/postgresql-cnpg/CHANGELOG.md b/charts/postgresql-cnpg/CHANGELOG.md new file mode 100644 index 0000000..63ca611 --- /dev/null +++ b/charts/postgresql-cnpg/CHANGELOG.md @@ -0,0 +1,125 @@ +# Changelog + +All notable changes to the `postgresql-cnpg` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.1] - 2026-09-01 + +### Added +- `examples/discovery-db.dev.yaml` - the discovery service's database. It pins + `dhi.io/pgvector:0.8-pg16`, because discovery-service's first migration + creates the `vector` extension and no stock CloudNativePG operand image + carries pgvector. + + Two things in it are load-bearing rather than decorative, and both were + verified against the image rather than assumed: + + - `postInitApplicationSQL` creates `vector` and `pg_trgm`. `pg_trgm` is + marked `trusted`, so a database owner may create it; **`vector` is not**, + and `CREATE EXTENSION vector` as a non-superuser fails with "permission + denied to create extension". The service connects as the bootstrap owner + and `enableSuperuserAccess` is false, so without this the first statement + of its migration fails and the pod crashloops on what looks like a + credentials problem. CNPG runs this hook as superuser in the newly created + database, which is the one moment superuser is available without granting + it to anything long-lived. + - `postgresUID`/`postgresGID` are 70, the image's postgres user. CNPG + defaults to 26 and applies it as the pod's `runAsUser` and `fsGroup`. + + The image is a general-purpose PostgreSQL image, not a CNPG operand image. It + meets CNPG's documented requirements (initdb, postgres, pg_ctl, + pg_controldata, pg_basebackup and du on PATH; CNPG overrides its entrypoint) + but has not been run under the operator, and it ships no barman-cloud - which + rules out the in-core backup method, though not the Barman Cloud Plugin. + + No template changed, so this affects no existing release. + +## [0.2.0] - 2026-08-31 + +### Added +- `databases`, rendering a CNPG `Database` object per entry, so one cluster can + host several service databases. `bootstrap.initdb` creates only one; this covers + the rest. Requires CNPG >= 1.25. +- `managed.roles`, rendering `spec.managed.roles` so the operator creates and + reconciles a role per application. Together with `databases` this lets every + service connect as the owner of its own database, with no workload holding + superuser rights. +- Render-time validation: a database listed under `databases` that duplicates + `bootstrap.database`, a database with no name or owner, and a managed role with + neither `passwordSecret` nor `disablePassword`. + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Adapted from an existing CloudNativePG chart, with the +deployment-specific and GCP-specific parts removed and the remainder made +configurable. + +### Added +- CNPG `Cluster`, plus optional Barman Cloud `ObjectStore` and + `ScheduledBackup`, and an optional ESO `ExternalSecret` for owner credentials. +- Depends on the `oan-common` library chart for names, labels and the mandatory + `resources` contract. +- Object store provider is selectable — `s3` (default), `gcs` or `azure` — with + the URI scheme derived from the provider and credentials passed through + verbatim as the matching `*Credentials` field. S3 defaults to + `inheritFromIAMRole`, so no keys are stored anywhere. +- Render-time validation of provider, bucket/destination, credential keys, + retention policy format, and logical-import configuration. +- `enableSuperuserAccess` (default false), `superuserSecret` and + `primaryUpdateStrategy` exposed. +- `extraClusterSpec` escape hatch for CNPG fields not yet exposed. +- Example per-environment values file for the registry database. + +### Fixed +- `nodeSelector` and `tolerations` now render inside `affinity`, where CNPG + expects them. Previously they were emitted at the top level of the `Cluster` + spec, which is not part of the CRD schema — the API server silently pruned + them, so pod scheduling constraints were never applied. +- `postgresUID` and `postgresGID` are now declared in `values.yaml`. The + templates referenced them but no schema defined them, so they could not be set + without an undocumented override. +- `bootstrap` is omitted entirely when nothing under it is configured, instead of + rendering a null `initdb`. +- `bootstrap.import.source.dbname` defaults to the single entry in + `import.databases` rather than rendering an empty string. + +### Changed +- `cluster.resources` moved to top-level `resources`, so the shared + `oan-common.resources` helper and its "requests and limits are mandatory" + guardrail apply. +- `cluster.instances`, `cluster.storage`, `cluster.walStorage`, + `cluster.postgresql`, `cluster.affinity`, `cluster.monitoring` and + `cluster.bootstrap` flattened to the top level; the `cluster` wrapper is gone. +- `cluster.name` replaced by the conventional `nameOverride` / + `fullnameOverride` pair. +- Backup ServiceAccount annotations moved from `backup.serviceAccountAnnotations` + to `serviceAccount.annotations`, since IRSA is not backup-specific. +- `ScheduledBackup`'s `immediate` and `backupOwnerReference` are configurable + rather than hardcoded. + +### Removed +- GCS-only object store, including the hardcoded `gs://` scheme, + `gkeEnvironment: true` and the GKE Workload Identity service account + annotation example. +- The `premium-rwo-retain` GKE storage class default; `storageClass` now defaults + to empty (the cluster's default class). +- References to the previous deployment's bundle layout, per-bundle values files, + Docker Hardened Images, and named example databases and users. +- Bitnami-specific framing of the logical import, which is now documented as a + generic migration from any existing PostgreSQL, and the `TODO-CONFIRM` markers + left in the original. +- `app.kubernetes.io/part-of: cloudnative-pg` label, replaced by the standard OAN + labels from `oan-common`. diff --git a/charts/postgresql-cnpg/Chart.yaml b/charts/postgresql-cnpg/Chart.yaml new file mode 100644 index 0000000..80b8352 --- /dev/null +++ b/charts/postgresql-cnpg/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: postgresql-cnpg +description: >- + CloudNativePG-managed PostgreSQL cluster for OAN services (Cluster + + optional ScheduledBackup and Barman Cloud ObjectStore). One release per + database. Requires the CloudNativePG operator; backups additionally require + the Barman Cloud Plugin, both cluster-wide installs. +type: application +version: 0.2.1 +# The PostgreSQL major line this chart targets by default. Unlike other OAN +# charts, this is not an image tag: when image.repository is empty the operator +# supplies its own default image. Pin image.* per environment for reproducibility. +appVersion: "17" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - postgresql + - postgres + - cloudnative-pg + - cnpg +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://cloudnative-pg.io/ +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/postgresql-cnpg/README.md b/charts/postgresql-cnpg/README.md new file mode 100644 index 0000000..eb46ab5 --- /dev/null +++ b/charts/postgresql-cnpg/README.md @@ -0,0 +1,220 @@ +# postgresql-cnpg + +A [CloudNativePG](https://cloudnative-pg.io/)-managed PostgreSQL cluster for OAN +services. One Helm release per database. + +This chart renders CNPG custom resources — a `Cluster`, and optionally an +`ObjectStore` and `ScheduledBackup` for backups. It does **not** render a +StatefulSet: the operator owns the pods, failover, and switchover. It keeps the +name `postgresql-cnpg` rather than an `oan-` prefix because it deploys +third-party software rather than an OAN-authored service. + +## Prerequisites + +Neither is installed by this chart, and both are cluster-wide: + +1. **The CloudNativePG operator.** Without it the rendered `Cluster` is an + unrecognised resource and nothing happens. +2. **The Barman Cloud Plugin** — only for backups. Leave `backup.enabled` false + until its CRDs are present. + +> **Status in OAN:** as of this chart's first release, neither is installed in +> `infra-automation`, and no IRSA role exists for the backup bucket. The S3 +> bucket does exist (`oan-dev-pg-backup`, versioned, 30-day expiry), and its +> Terraform comment notes the IRSA role is deliberately deferred until the +> Postgres operator was chosen — this chart is that choice. Until the operator +> and role land, this chart renders and lints but cannot be usefully installed. + +## Install + +```bash +helm dependency update charts/postgresql-cnpg +helm install registry-db charts/postgresql-cnpg \ + -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml +``` + +See [`examples/registry-db.dev.yaml`](./examples/registry-db.dev.yaml) for a +complete, commented per-environment values file. + +## Databases and roles + +A CNPG cluster's `bootstrap.initdb` creates **exactly one** database. Everything +else is a `Database` object the operator reconciles, so one cluster can host +several service databases without any of them sharing tables or credentials. + +```yaml +# the first database, created when the cluster is bootstrapped +bootstrap: + database: registry + owner: registry + ownerSecret: registry-db-app + +# roles for the others, created and reconciled by the operator +managed: + roles: + - name: keycloak + passwordSecret: + name: keycloak-db # Secret with `username` and `password` + +# the others +databases: + - name: keycloak + owner: keycloak +``` + +That renders `Cluster/registry-db` plus `Database/registry-db-keycloak`, giving: + +| Database | Owner | Created by | +|---|---|---| +| `registry` | `registry` | `bootstrap.initdb` | +| `keycloak` | `keycloak` | `Database` object | + +**The point of the separation:** every application connects as the owner of its +own database, so no workload needs superuser rights, and one service cannot read +another's tables. `enableSuperuserAccess` stays at the CNPG default of `false`. + +> **Requires CNPG >= 1.25**, where the `Database` CRD was introduced. On an older +> operator these objects are ignored *silently* — no database, and no error. If a +> database does not appear, check the operator version first. + +Each entry also supports `ensure` (`present`/`absent`), `databaseReclaimPolicy` +(defaults to `retain`, so deleting the object leaves the data), `encoding`, +`locale`, `template`, `allowConnections`, `connectionLimit`, `extensions` and +`schemas`. + +The render fails if you list the bootstrap database under `databases` — the +operator would be asked to create one that already exists — or if a database +names an owner with no `passwordSecret` and no `disablePassword`. + +## Connecting to it + +CNPG creates three services from the cluster name: + +| Service | Points at | +|---|---| +| `-rw` | The primary. Read-write. What applications normally use. | +| `-ro` | Replicas only. Read-only. | +| `-r` | Any instance, primary included. Read-only workloads that tolerate the primary. | + +**Set `fullnameOverride`.** The cluster name becomes the DNS other services +depend on, and renaming a cluster later means recreating it. With +`fullnameOverride: registry-db` the primary is +`registry-db-rw..svc.cluster.local:5432`; without it you get +`-postgresql-cnpg-rw`. + +## Credentials + +The owner password can come from three places: + +1. **CNPG generates it** — leave `bootstrap.ownerSecret` empty. Read it from + `Secret/-app`. Fine for local and dev. +2. **A Secret you create** — set `bootstrap.ownerSecret` to its name. It must + carry `username` and `password` keys. +3. **A Secret managed by whatever handles secrets in that environment** — point + `bootstrap.ownerSecret` at it. This chart renders no Secrets; it only + references them. + +Superuser access is off by default, matching the CNPG default: applications +connect as the database owner, not as `postgres`. + +## Backups + +Base backups plus continuous WAL archiving through the Barman Cloud Plugin, +which together give point-in-time recovery. Off by default. + +To enable, in this order: + +1. Install the Barman Cloud Plugin cluster-wide. +2. Set `backup.objectStore.bucket` (and `path` if several clusters share the + bucket). +3. Grant access. On EKS that is IRSA: an IAM role trusted by the cluster's + ServiceAccount, named in `serviceAccount.annotations` as + `eks.amazonaws.com/role-arn`. The default credentials block is + `{inheritFromIAMRole: true}`, so no keys go anywhere near the chart. +4. Set `backup.enabled: true`. + +`backup.retentionPolicy` must be **shorter** than any expiry the bucket's own +lifecycle policy applies. The OAN dev bucket expires objects after 30 days, so +the chart's `14d` default sits safely inside it. Get this backwards and S3 +deletes backups Barman still believes it has. + +`provider` supports `s3`, `gcs` and `azure`; `credentials` is passed through +verbatim as the matching `s3Credentials` / `googleCredentials` / +`azureCredentials` field. Because Helm merges maps, switching provider does not +remove the S3 default — the render fails and tells you to null it: + +```bash +--set backup.objectStore.provider=gcs \ +--set backup.objectStore.credentials.inheritFromIAMRole=null \ +--set backup.objectStore.credentials.gkeEnvironment=true +``` + +## Storage + +`storageClass` empty means the cluster's default class. On the OAN clusters that +is `gp3` with **`reclaimPolicy: Delete`**, which deletes the underlying EBS +volume when the PVC goes away. For any database you would miss, point +`storage.storageClass` and `walStorage.storageClass` at a Retain class. + +`walStorage` is a separate volume for the write-ahead log, enabled by default. +It is effectively required for healthy PITR throughput. + +## Migrating an existing database in + +`bootstrap.import` runs CNPG's logical import (`pg_dump`/`pg_restore`) against an +existing PostgreSQL when the cluster is **first created**. It has no effect on an +existing cluster, so it is a one-shot cutover tool, not a sync. + +```yaml +bootstrap: + database: registry + owner: registry + import: + enabled: true + databases: [registry] + source: + host: old-postgres.example + passwordSecret: + name: old-postgres-superuser +``` + +`source.dbname` defaults to the single entry in `databases`. + +## Render-time validation + +The chart fails the render rather than letting a misconfiguration reach the +cluster: + +| Condition | Why it matters | +|---|---| +| `resources` empty | Every OAN component must declare requests and limits (`oan-common.resources`) | +| `backup.enabled` with no `bucket` or `destinationPath` | The ObjectStore would have nowhere to write | +| `backup.objectStore.provider` not `s3`/`gcs`/`azure` | Would render an unknown credentials field | +| A credentials key not valid for the chosen provider | Catches the Helm map-merge trap described above | +| `retentionPolicy` not matching `^[1-9][0-9]*[dwm]$` | The CRD rejects it; `"14days"` fails here instead of at apply | +| `import.enabled` without `source.host` or `source.passwordSecret.name` | CNPG could not read the source | +| `import.databases` not exactly one entry | `type: microservice` permits one database | + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema. CNPG fields the +chart does not expose yet can be passed through `extraClusterSpec`, which is +merged into the `Cluster` spec. + +Note on `appVersion`: for this chart it documents the PostgreSQL major line +targeted by default rather than a pullable tag, because leaving `image.repository` +empty lets the operator supply its own default image. Pin `image.*` per +environment — a PostgreSQL major upgrade is a data migration, not a tag bump. + +## Known gaps + +- **No restore path.** `bootstrap.recovery` (restoring a new cluster from an + object store) is not implemented. Backups are only half a disaster-recovery + story; this needs its own change once backups are actually running. +- **No PodMonitor by default.** `monitoring.enablePodMonitor` requires the + Prometheus Operator CRDs, which are not installed yet. + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/postgresql-cnpg/ci/lint-values.yaml b/charts/postgresql-cnpg/ci/lint-values.yaml new file mode 100644 index 0000000..05874c6 --- /dev/null +++ b/charts/postgresql-cnpg/ci/lint-values.yaml @@ -0,0 +1,10 @@ +# Exercises the backup and ESO paths in CI, which are off in the defaults. +fullnameOverride: ci-db +bootstrap: + database: ci + owner: ci +backup: + enabled: true + objectStore: + bucket: ci-bucket + path: /ci diff --git a/charts/postgresql-cnpg/examples/discovery-db.dev.yaml b/charts/postgresql-cnpg/examples/discovery-db.dev.yaml new file mode 100644 index 0000000..e876898 --- /dev/null +++ b/charts/postgresql-cnpg/examples/discovery-db.dev.yaml @@ -0,0 +1,139 @@ +# Example: the discovery service's database, dev environment. +# +# helm install discovery-db charts/postgresql-cnpg \ +# -n oan-discovery -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml +# +# The discovery service then connects to: +# discovery-db-rw.oan-discovery.svc.cluster.local:5432 +# +# Separate from registry-db in every way - its own cluster, its own namespace, +# its own PostgreSQL major. Nothing here affects the registry stack. + +fullnameOverride: discovery-db + +# --------------------------------------------------------------------------- +# Image +# +# discovery-service needs pgvector, which the stock CloudNativePG operand image +# does not carry. This is the Docker Hardened Image of pgvector: pgvector 0.8.6 +# and pg_trgm 1.6 on PostgreSQL 16, published for both amd64 and arm64 (the OAN +# nodes are arm64). +# +# pg16 rather than the pg14 tag, because PostgreSQL 16 is what the service is +# actually verified against: its README and design doc name it, its +# testcontainers suite pins pgvector/pgvector:0.8.0-pg16, and every measured +# planner claim in the design doc - the 20 jsonpath shapes GIN captures, the +# index selection over 300k rows - was measured on 16. tests/dbtest also uses +# EXPLAIN (GENERIC_PLAN), which is PostgreSQL 16 and later only. +# +# The schema itself does apply cleanly to 14, so this is about staying on the +# major the service was tested on rather than about a hard incompatibility. +# +# CAVEAT, and it is the one to smoke-test first: this is a general-purpose +# PostgreSQL image, not a CNPG operand image. It satisfies CNPG's documented +# requirements - initdb, postgres, pg_ctl, pg_controldata, pg_basebackup and du +# are all on PATH, and CNPG overrides the image's own entrypoint - but it is +# not built from cloudnative-pg/postgres-containers and has not been run under +# the operator here. Verify the first cluster reaches "Cluster in healthy +# state" before pointing anything at it. +# +# It also ships no barman-cloud. That costs nothing while backups run through +# the Barman Cloud Plugin, which supplies its own sidecar, and nothing at all +# while backup.enabled is false - but the in-core backup method is not an +# option with this image. +image: + registry: dhi.io + repository: pgvector + tag: "0.8-pg16" + +# The image's postgres user is uid/gid 70. CNPG defaults to 26 and applies it +# as the pod's runAsUser and fsGroup, so leaving these unset gives the data +# directory to a user that does not exist in this image. +postgresUID: 70 +postgresGID: 70 + +# Dev: single instance. Use 2+ where an outage matters. +instances: 1 + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +storage: + size: 20Gi + # Empty uses the cluster default (gp3, reclaimPolicy Delete). Point this at a + # Retain class before this database holds anything worth keeping. + storageClass: "" + +walStorage: + enabled: true + size: 2Gi + +bootstrap: + database: discovery + owner: discovery + # Omit ownerSecret and CNPG generates Secret/discovery-db-app itself, with a + # ready-made `uri` key the discovery chart consumes through + # database.urlSecret. Simplest correct thing for dev - nothing to create by + # hand and no password to escape. + ownerSecret: "" + + # --------------------------------------------------------------------------- + # This is load-bearing, not a convenience. + # + # discovery-service's first migration opens with: + # CREATE EXTENSION IF NOT EXISTS vector; + # CREATE EXTENSION IF NOT EXISTS pg_trgm; + # + # pg_trgm is marked `trusted = true`, so the database owner may create it. + # `vector` is NOT trusted, and creating it requires superuser: + # + # ERROR: permission denied to create extension "vector" + # HINT: Must be superuser to create this extension. + # + # The service connects as `discovery`, the bootstrap owner, and + # enableSuperuserAccess is false below - so with these lines absent, the very + # first statement of the migration fails and the pod crashloops on an error + # that reads like a bad password rather than a missing grant. + # + # CNPG runs postInitApplicationSQL as superuser inside the freshly created + # application database, which is the one moment superuser is available + # without granting it to anything long-lived. Afterwards the service's own + # `IF NOT EXISTS` is a no-op and the rest of the migration runs as the owner. + # + # Verified against dhi.io/pgvector: as the non-superuser owner the migration + # fails on statement one without these, and applies in full with them. + # --------------------------------------------------------------------------- + postInitApplicationSQL: + - CREATE EXTENSION IF NOT EXISTS vector + - CREATE EXTENSION IF NOT EXISTS pg_trgm + +# The service owns its own database, so nothing here needs superuser rights +# beyond the bootstrap moment above. +enableSuperuserAccess: false + +# One database, one owner - no managed.roles and no extra `databases` entries. +# Unlike registry-db, nothing else shares this cluster. + +# Off until the Barman Cloud Plugin is installed and an IRSA role exists. +# Note the image ships no barman-cloud of its own, so the plugin - which brings +# its own sidecar - is the only backup route here. +backup: + enabled: false + retentionPolicy: "14d" + objectStore: + bucket: oan-dev-pg-backup + path: /discovery + credentials: + inheritFromIAMRole: true + +serviceAccount: + annotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/oan-dev-pg-backup + +commonLabels: + oan.in/environment: dev diff --git a/charts/postgresql-cnpg/examples/registry-db.dev.yaml b/charts/postgresql-cnpg/examples/registry-db.dev.yaml new file mode 100644 index 0000000..2f950d0 --- /dev/null +++ b/charts/postgresql-cnpg/examples/registry-db.dev.yaml @@ -0,0 +1,100 @@ +# Example: the registry service's database, dev environment. +# +# helm install registry-db charts/postgresql-cnpg \ +# -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml +# +# Other workloads then connect to: +# registry-db-rw.oan-registry.svc.cluster.local:5432 + +fullnameOverride: registry-db + +# Sunbird RC's registry is verified against PostgreSQL 14. Pin it rather than +# inheriting the operator default, so an operator upgrade cannot move the major. +image: + registry: ghcr.io + repository: cloudnative-pg/postgresql + tag: "14.13" + +# Dev: single instance. Use 2+ where an outage matters. +instances: 1 + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +storage: + size: 10Gi + # Empty uses the cluster default (gp3, reclaimPolicy Delete). Point this at a + # Retain class before this database holds anything worth keeping. + storageClass: "" + +walStorage: + enabled: true + size: 2Gi + +# The registry's database, created by initdb when the cluster is bootstrapped. +# A cluster gets exactly one of these; everything else goes under `databases`. +bootstrap: + database: registry + owner: registry + # Secret carrying `username` and `password` for the owner role. + # kubectl -n oan-registry create secret generic registry-db-app \ + # --from-literal=username=registry --from-literal=password='' + # Or drop ownerSecret and let CNPG generate the password itself. + ownerSecret: registry-db-app + +# Keycloak's own role. The operator creates and reconciles it, so Keycloak never +# needs superuser rights. +# kubectl -n oan-registry create secret generic keycloak-db \ +# --from-literal=username=keycloak --from-literal=password='' +managed: + roles: + - name: keycloak + passwordSecret: + name: keycloak-db + +# Keycloak's own database, owned by that role. Keeping its realm tables out of +# the registry's database is the one place this deliberately improves on the +# compose stack, which shares a single `registry` database between both services. +# +# REQUIRES CNPG >= 1.25 for the Database CRD. +databases: + - name: keycloak + owner: keycloak + +# Left at the CNPG default (false). Each service connects as the owner of its own +# database, so nothing here needs superuser rights. +# +# The migration chart is the exception: it wants rights across every database it +# migrates. Set this true if you enable migration targets that actually have SQL. +enableSuperuserAccess: false + +# Off until the Barman Cloud Plugin is installed and the IRSA role for +# oan-dev-pg-backup exists (see infra-automation: buckets.pg_backup). +backup: + enabled: false + retentionPolicy: "14d" + objectStore: + bucket: oan-dev-pg-backup + path: /registry + credentials: + inheritFromIAMRole: true + +# The IRSA role the cluster pods assume in order to write those backups. +serviceAccount: + annotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/oan-dev-pg-backup + +# The owner Secret above is created out of band. Until a secrets manager is +# wired up, either create it by hand: +# kubectl -n oan-registry create secret generic registry-db-app \ +# --from-literal=username=registry --from-literal=password='' +# or drop bootstrap.ownerSecret entirely and let CNPG generate the password into +# Secret/registry-db-app itself, which is simpler for dev. + +commonLabels: + oan.in/environment: dev diff --git a/charts/postgresql-cnpg/templates/NOTES.txt b/charts/postgresql-cnpg/templates/NOTES.txt new file mode 100644 index 0000000..16c304f --- /dev/null +++ b/charts/postgresql-cnpg/templates/NOTES.txt @@ -0,0 +1,41 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +CloudNativePG resources in namespace "{{ include "postgresql-cnpg.namespace" . }}": + - Cluster/{{ include "postgresql-cnpg.fullname" . }} ({{ .Values.instances }} instance(s)) +{{- if .Values.backup.enabled }} + - ObjectStore/{{ include "postgresql-cnpg.objectStoreName" . }} -> {{ include "postgresql-cnpg.objectStore.destinationPath" . }} + - ScheduledBackup/{{ include "postgresql-cnpg.fullname" . }}-scheduled ({{ .Values.backup.schedule }}, retention {{ .Values.backup.retentionPolicy }}) +{{- end }} + +Connect from another workload in the cluster: + host: {{ include "postgresql-cnpg.fullname" . }}-rw.{{ include "postgresql-cnpg.namespace" . }}.svc.cluster.local # primary, read-write + host: {{ include "postgresql-cnpg.fullname" . }}-ro.{{ include "postgresql-cnpg.namespace" . }}.svc.cluster.local # replicas, read-only + port: 5432 +{{- with .Values.bootstrap.database }} + database: {{ . }} +{{- end }} +{{- with .Values.bootstrap.owner }} + user: {{ . }} +{{- end }} + +{{ if not .Values.fullnameOverride }} +NOTE: fullnameOverride is not set, so the cluster is named +"{{ include "postgresql-cnpg.fullname" . }}" and other services must connect to +"{{ include "postgresql-cnpg.fullname" . }}-rw". Set fullnameOverride (e.g. +"registry-db") for a shorter, stable name - renaming later means recreating the +cluster. +{{- end }} +{{- if not .Values.bootstrap.ownerSecret }} + +NOTE: bootstrap.ownerSecret is not set, so CNPG generated the owner password +itself. Read it with: + kubectl -n {{ include "postgresql-cnpg.namespace" . }} get secret {{ include "postgresql-cnpg.fullname" . }}-app -o jsonpath='{.data.password}' | base64 -d +{{- end }} +{{- if not .Values.backup.enabled }} + +NOTE: backups are disabled. Enable backup.* once the Barman Cloud Plugin is +installed cluster-wide and the bucket and IRSA role exist. +{{- end }} + +Watch the cluster come up: + kubectl -n {{ include "postgresql-cnpg.namespace" . }} get cluster {{ include "postgresql-cnpg.fullname" . }} -w diff --git a/charts/postgresql-cnpg/templates/_helpers.tpl b/charts/postgresql-cnpg/templates/_helpers.tpl new file mode 100644 index 0000000..901143c --- /dev/null +++ b/charts/postgresql-cnpg/templates/_helpers.tpl @@ -0,0 +1,110 @@ +{{/* +# ============================================================================ +# OAN POSTGRESQL CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the CNPG-specific +# naming and object-store logic this chart needs. +# ============================================================================ +*/}} + +{{/* +Cluster name. Drives the CNPG service names -rw / -ro / -r, +which other charts connect to, so it should be short and stable - set +fullnameOverride (e.g. "registry-db") rather than relying on -. +*/}} +{{- define "postgresql-cnpg.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "postgresql-cnpg.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "postgresql-cnpg.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{/* +Namespace for the CNPG resources. Defaults to the release namespace. +*/}} +{{- define "postgresql-cnpg.namespace" -}} +{{- default (include "oan-common.namespace" .) .Values.namespace -}} +{{- end }} + +{{/* +Image reference, or empty when no image is configured - in which case the +Cluster omits imageName and the operator uses its own default image. +*/}} +{{- define "postgresql-cnpg.image" -}} +{{- if .Values.image.repository -}} +{{- include "oan-common.image" . -}} +{{- end -}} +{{- end }} + +{{/* +Barman ObjectStore name (defaults to -backup). +*/}} +{{- define "postgresql-cnpg.objectStoreName" -}} +{{- default (printf "%s-backup" (include "postgresql-cnpg.fullname" .)) .Values.backup.objectStore.name -}} +{{- end }} + +{{/* +Map the object store provider to its CNPG credentials field. +*/}} +{{- define "postgresql-cnpg.objectStore.credentialsField" -}} +{{- $field := dict "s3" "s3Credentials" "gcs" "googleCredentials" "azure" "azureCredentials" -}} +{{- index $field .Values.backup.objectStore.provider -}} +{{- end }} + +{{/* +Object store destination path. An explicit destinationPath wins; otherwise it is +built from the provider's URI scheme plus bucket and path. +*/}} +{{- define "postgresql-cnpg.objectStore.destinationPath" -}} +{{- $os := .Values.backup.objectStore -}} +{{- if $os.destinationPath -}} +{{- $os.destinationPath -}} +{{- else -}} +{{- $scheme := index (dict "s3" "s3://" "gcs" "gs://" "azure" "azure://") $os.provider -}} +{{- printf "%s%s%s" $scheme $os.bucket ($os.path | default "") -}} +{{- end -}} +{{- end }} + +{{/* +Validate the backup configuration. Called from the templates that need it so a +misconfiguration fails the render rather than producing an ObjectStore the +plugin silently cannot use. +*/}} +{{- define "postgresql-cnpg.validateBackup" -}} +{{- $os := .Values.backup.objectStore -}} +{{- $providers := list "s3" "gcs" "azure" -}} +{{- if not (has $os.provider $providers) -}} +{{- fail (printf "%s: backup.objectStore.provider must be one of %s, got %q" .Chart.Name (join ", " $providers) $os.provider) -}} +{{- end -}} +{{- if and (not $os.bucket) (not $os.destinationPath) -}} +{{- fail (printf "%s: backup.enabled is true but neither backup.objectStore.bucket nor backup.objectStore.destinationPath is set" .Chart.Name) -}} +{{- end -}} +{{- if not $os.credentials -}} +{{- fail (printf "%s: backup.enabled is true but backup.objectStore.credentials is empty. For S3 with IRSA use {inheritFromIAMRole: true}" .Chart.Name) -}} +{{- end -}} +{{/* +Credentials keys must belong to the chosen provider. Helm merges maps, so +switching provider away from the s3 default would otherwise carry +`inheritFromIAMRole` into googleCredentials, where the plugin rejects it. +*/}} +{{- $allowed := index (dict + "s3" (list "accessKeyId" "inheritFromIAMRole" "region" "secretAccessKey" "sessionToken") + "gcs" (list "applicationCredentials" "gkeEnvironment") + "azure" (list "connectionString" "inheritFromAzureAD" "storageAccount" "storageKey" "storageSasToken" "useDefaultAzureCredentials") + ) $os.provider -}} +{{- range $key, $_ := $os.credentials -}} +{{- if not (has $key $allowed) -}} +{{- fail (printf "%s: backup.objectStore.credentials.%s is not valid for provider %q (allowed: %s). If it came from this chart's default, null it out: --set backup.objectStore.credentials.%s=null" $.Chart.Name $key $os.provider (join ", " $allowed) $key) -}} +{{- end -}} +{{- end -}} +{{- with .Values.backup.retentionPolicy -}} +{{- if not (regexMatch "^[1-9][0-9]*[dwm]$" .) -}} +{{- fail (printf "backup.retentionPolicy must match ^[1-9][0-9]*[dwm]$ (days, weeks or months, e.g. \"14d\"), got %q" .) -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/charts/postgresql-cnpg/templates/cluster.yaml b/charts/postgresql-cnpg/templates/cluster.yaml new file mode 100644 index 0000000..01d1123 --- /dev/null +++ b/charts/postgresql-cnpg/templates/cluster.yaml @@ -0,0 +1,202 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ include "postgresql-cnpg.fullname" . }} + namespace: {{ include "postgresql-cnpg.namespace" . }} + labels: + {{- include "postgresql-cnpg.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + instances: {{ .Values.instances }} + {{- with (include "postgresql-cnpg.image" .) }} + imageName: {{ . | quote }} + {{- end }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.postgresUID }} + postgresUID: {{ . }} + {{- end }} + {{- with .Values.postgresGID }} + postgresGID: {{ . }} + {{- end }} + primaryUpdateStrategy: {{ .Values.primaryUpdateStrategy }} + enableSuperuserAccess: {{ .Values.enableSuperuserAccess }} + {{- with .Values.superuserSecret }} + superuserSecret: + name: {{ . }} + {{- end }} + + {{- with .Values.serviceAccount.annotations }} + serviceAccountTemplate: + metadata: + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + + storage: + size: {{ .Values.storage.size }} + {{- with .Values.storage.storageClass }} + storageClass: {{ . }} + {{- end }} + {{- if .Values.walStorage.enabled }} + walStorage: + size: {{ .Values.walStorage.size }} + {{- with .Values.walStorage.storageClass }} + storageClass: {{ . }} + {{- end }} + {{- end }} + + resources: + {{- include "oan-common.resources" . | nindent 4 }} + + {{- $affinity := omit .Values.affinity "nodeSelector" "tolerations" }} + {{- with .Values.affinity.nodeSelector }} + {{- $affinity = set $affinity "nodeSelector" . }} + {{- end }} + {{- with .Values.affinity.tolerations }} + {{- $affinity = set $affinity "tolerations" . }} + {{- end }} + {{- with $affinity }} + # CNPG nests nodeSelector and tolerations inside affinity; the Cluster spec + # has no top-level fields for them, and anything put there is silently + # dropped by the API server. + affinity: + {{- toYaml . | nindent 4 }} + {{- end }} + + {{- with .Values.managed.roles }} + {{/* + Roles created and reconciled by the operator. A database owner must exist + before the Database object referencing it can be applied, so any role named as + an owner under `databases` belongs here. + */}} + managed: + roles: + {{- range . }} + {{- if not .name }} + {{- fail (printf "%s: every entry in managed.roles needs a name" $.Chart.Name) }} + {{- end }} + - name: {{ .name | quote }} + ensure: {{ .ensure | default "present" }} + login: {{ .login | default true }} + {{- if hasKey . "superuser" }} + superuser: {{ .superuser }} + {{- end }} + {{- if hasKey . "createdb" }} + createdb: {{ .createdb }} + {{- end }} + {{- if hasKey . "createrole" }} + createrole: {{ .createrole }} + {{- end }} + {{- if hasKey . "inherit" }} + inherit: {{ .inherit }} + {{- end }} + {{- with .connectionLimit }} + connectionLimit: {{ . }} + {{- end }} + {{- with .inRoles }} + inRoles: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .comment }} + comment: {{ . | quote }} + {{- end }} + {{- if .passwordSecret }} + passwordSecret: + name: {{ .passwordSecret.name }} + {{- else if .disablePassword }} + disablePassword: true + {{- else }} + {{- fail (printf "%s: managed role %q has no passwordSecret. Set one (a Secret with username and password keys), or set disablePassword: true if the role should not log in with a password." $.Chart.Name .name) }} + {{- end }} + {{- end }} + {{- end }} + + {{- with .Values.postgresql.parameters }} + postgresql: + parameters: + {{- toYaml . | nindent 6 }} + {{- end }} + + {{- if .Values.monitoring.enablePodMonitor }} + monitoring: + enablePodMonitor: true + {{- end }} + + {{- if .Values.backup.enabled }} + {{- include "postgresql-cnpg.validateBackup" . }} + # Base backups and WAL archiving via the Barman Cloud Plugin (CNPG-I). + plugins: + - name: {{ .Values.backup.pluginName }} + isWALArchiver: true + parameters: + barmanObjectName: {{ include "postgresql-cnpg.objectStoreName" . }} + {{- end }} + + {{- $boot := .Values.bootstrap }} + {{- if or $boot.database $boot.owner $boot.ownerSecret $boot.postInitApplicationSQL $boot.import.enabled }} + bootstrap: + initdb: + {{- with .Values.bootstrap.database }} + database: {{ . | quote }} + {{- end }} + {{- with .Values.bootstrap.owner }} + owner: {{ . | quote }} + {{- end }} + {{- with .Values.bootstrap.ownerSecret }} + secret: + name: {{ . }} + {{- end }} + {{- if .Values.bootstrap.import.enabled }} + import: + type: {{ .Values.bootstrap.import.type }} + databases: + {{- toYaml .Values.bootstrap.import.databases | nindent 10 }} + source: + externalCluster: {{ include "postgresql-cnpg.fullname" . }}-source + {{- with .Values.bootstrap.postInitApplicationSQL }} + postImportApplicationSQL: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- else }} + {{- with .Values.bootstrap.postInitApplicationSQL }} + postInitApplicationSQL: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} + + {{- if .Values.bootstrap.import.enabled }} + {{- $src := .Values.bootstrap.import.source }} + {{- if not $src.host }} + {{- fail (printf "%s: bootstrap.import.enabled is true but bootstrap.import.source.host is empty" .Chart.Name) }} + {{- end }} + {{- if not (eq (len .Values.bootstrap.import.databases) 1) }} + {{- fail (printf "%s: bootstrap.import.type microservice needs exactly one entry in bootstrap.import.databases, got %d" .Chart.Name (len .Values.bootstrap.import.databases)) }} + {{- end }} + {{- if not $src.passwordSecret.name }} + {{- fail (printf "%s: bootstrap.import.enabled is true but bootstrap.import.source.passwordSecret.name is empty - CNPG needs credentials to read the source" .Chart.Name) }} + {{- end }} + # The existing PostgreSQL this cluster is imported from, read logically at + # creation time only. + externalClusters: + - name: {{ include "postgresql-cnpg.fullname" . }}-source + connectionParameters: + host: {{ $src.host | quote }} + port: {{ $src.port | quote }} + dbname: {{ $src.dbname | default (first .Values.bootstrap.import.databases) | quote }} + user: {{ $src.user | quote }} + password: + name: {{ $src.passwordSecret.name }} + key: {{ $src.passwordSecret.key }} + {{- end }} + {{- with .Values.extraClusterSpec }} + {{- toYaml . | nindent 2 }} + {{- end }} diff --git a/charts/postgresql-cnpg/templates/database.yaml b/charts/postgresql-cnpg/templates/database.yaml new file mode 100644 index 0000000..896845b --- /dev/null +++ b/charts/postgresql-cnpg/templates/database.yaml @@ -0,0 +1,64 @@ +{{- range .Values.databases }} +{{- if not .name }} +{{- fail (printf "%s: every entry in databases needs a name" $.Chart.Name) }} +{{- end }} +{{- if not .owner }} +{{- fail (printf "%s: database %q needs an owner. The role must exist - declare it under managed.roles, or use the cluster's bootstrap owner." $.Chart.Name .name) }} +{{- end }} +{{- if eq .name ($.Values.bootstrap.database | default "") }} +{{- fail (printf "%s: database %q is already created by bootstrap.database. Remove it from `databases`, or rename one of them - the operator would otherwise be asked to create a database that already exists." $.Chart.Name .name) }} +{{- end }} +--- +{{/* +A database created declaratively by the CNPG operator, alongside the single +database that bootstrap.initdb creates. + +Requires CNPG >= 1.25, which is where the Database CRD was introduced. On an +older operator this object is simply ignored - no database, no error - so check +the operator version if a database does not appear. +*/}} +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: {{ printf "%s-%s" (include "postgresql-cnpg.fullname" $) .name | trunc 63 | trimSuffix "-" }} + namespace: {{ include "postgresql-cnpg.namespace" $ }} + labels: + {{- include "postgresql-cnpg.labels" $ | nindent 4 }} + oan.in/database: {{ .name | quote }} + {{- with $.Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + cluster: + name: {{ include "postgresql-cnpg.fullname" $ }} + name: {{ .name | quote }} + owner: {{ .owner | quote }} + ensure: {{ .ensure | default "present" }} + # retain: dropping this object leaves the database in place. Databases hold + # data that outlives a Helm release, so deletion is deliberately not automatic. + databaseReclaimPolicy: {{ .databaseReclaimPolicy | default "retain" }} + {{- with .encoding }} + encoding: {{ . | quote }} + {{- end }} + {{- with .locale }} + locale: {{ . | quote }} + {{- end }} + {{- with .template }} + template: {{ . | quote }} + {{- end }} + {{- if hasKey . "allowConnections" }} + allowConnections: {{ .allowConnections }} + {{- end }} + {{- with .connectionLimit }} + connectionLimit: {{ . }} + {{- end }} + {{- with .extensions }} + extensions: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .schemas }} + schemas: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/postgresql-cnpg/templates/objectstore.yaml b/charts/postgresql-cnpg/templates/objectstore.yaml new file mode 100644 index 0000000..bf2a86f --- /dev/null +++ b/charts/postgresql-cnpg/templates/objectstore.yaml @@ -0,0 +1,41 @@ +{{- if .Values.backup.enabled }} +{{- include "postgresql-cnpg.validateBackup" . }} +{{- $os := .Values.backup.objectStore }} +# Barman Cloud Plugin ObjectStore - target for base backups and WAL archiving. +# Requires the Barman Cloud Plugin installed cluster-wide. +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: {{ include "postgresql-cnpg.objectStoreName" . }} + namespace: {{ include "postgresql-cnpg.namespace" . }} + labels: + {{- include "postgresql-cnpg.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.backup.retentionPolicy }} + retentionPolicy: {{ . | quote }} + {{- end }} + configuration: + destinationPath: {{ include "postgresql-cnpg.objectStore.destinationPath" . | quote }} + {{- with $os.endpointURL }} + endpointURL: {{ . | quote }} + {{- end }} + {{- with $os.serverName }} + serverName: {{ . | quote }} + {{- end }} + {{ include "postgresql-cnpg.objectStore.credentialsField" . }}: + {{- toYaml $os.credentials | nindent 6 }} + wal: + compression: {{ $os.wal.compression }} + {{- with $os.wal.encryption }} + encryption: {{ . }} + {{- end }} + data: + compression: {{ $os.data.compression }} + {{- with $os.data.encryption }} + encryption: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/postgresql-cnpg/templates/scheduledbackup.yaml b/charts/postgresql-cnpg/templates/scheduledbackup.yaml new file mode 100644 index 0000000..252b2ee --- /dev/null +++ b/charts/postgresql-cnpg/templates/scheduledbackup.yaml @@ -0,0 +1,27 @@ +{{- if .Values.backup.enabled }} +# Scheduled base backups. WAL is archived continuously by the plugin, so these +# plus the archived WAL are what make point-in-time recovery possible. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: {{ include "postgresql-cnpg.fullname" . }}-scheduled + namespace: {{ include "postgresql-cnpg.namespace" . }} + labels: + {{- include "postgresql-cnpg.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + # 6-field cron: sec min hour dom mon dow + schedule: {{ .Values.backup.schedule | quote }} + backupOwnerReference: {{ .Values.backup.backupOwnerReference }} + immediate: {{ .Values.backup.immediate }} + method: plugin + pluginConfiguration: + name: {{ .Values.backup.pluginName }} + parameters: + barmanObjectName: {{ include "postgresql-cnpg.objectStoreName" . }} + cluster: + name: {{ include "postgresql-cnpg.fullname" . }} +{{- end }} diff --git a/charts/postgresql-cnpg/values.yaml b/charts/postgresql-cnpg/values.yaml new file mode 100644 index 0000000..68c3958 --- /dev/null +++ b/charts/postgresql-cnpg/values.yaml @@ -0,0 +1,271 @@ +# ============================================================================ +# postgresql-cnpg - default values +# +# One CloudNativePG `Cluster` (plus optional backups) per database. Install one +# release per service that needs its own database, giving each its own +# credentials, storage and backup schedule: +# +# helm install registry-db charts/postgresql-cnpg \ +# -n oan-registry -f values/registry-db.dev.yaml +# +# PREREQUISITES (neither is installed by this chart): +# 1. The CloudNativePG operator, cluster-wide. +# 2. For backups only: the Barman Cloud Plugin, also cluster-wide. Leave +# backup.enabled false until its CRDs are present. +# +# Minimum you must set for a real deployment: +# - fullnameOverride (see below - it decides the DNS other services use) +# - bootstrap.database / bootstrap.owner / bootstrap.ownerSecret +# - storage.size and storage.storageClass +# ============================================================================ + +# Namespace for the CNPG resources. Defaults to the release namespace; set this +# only if you deliberately want them somewhere other than `helm -n`. +namespace: "" + +nameOverride: "" +# STRONGLY RECOMMENDED. The cluster name becomes the connection DNS other +# services use - -rw (primary), -ro (replicas), -r (any). +# Without this you get -postgresql-cnpg-rw. Set something short and +# stable, e.g. "registry-db" -> registry-db-rw..svc.cluster.local +fullnameOverride: "" + +# --------------------------------------------------------------------------- +# Image +# Leave repository empty to let the CNPG operator choose its own default +# PostgreSQL image. Pin it per environment for reproducible upgrades - a +# PostgreSQL major version change is a data migration, not a tag bump. +# --------------------------------------------------------------------------- +image: + # e.g. "ghcr.io" + registry: dhi.io + # e.g. "cloudnative-pg/postgresql" + repository: pgvector + tag: "0.8-pg16" + # sha256:... - wins over tag when set + digest: "" + pullSecrets: [] + +# Primary + (instances - 1) replicas. 1 is fine for local or dev; use at least +# 2 anywhere an outage matters. +instances: 2 + +# Resource requests and limits for each instance. REQUIRED - the render fails +# when this is empty (oan-common.resources). +resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +# --------------------------------------------------------------------------- +# Storage +# storageClass empty means the cluster's default class. Note the OAN clusters +# default to gp3 with reclaimPolicy: Delete - for anything you cannot lose, +# point this at a Retain class so the volume survives PVC deletion. +# --------------------------------------------------------------------------- +storage: + size: 10Gi + storageClass: "" + +# Separate volume for the write-ahead log. Recommended, and effectively +# required for healthy point-in-time-recovery throughput. +walStorage: + enabled: true + size: 2Gi + storageClass: "" + +# PostgreSQL server parameters, merged onto the CNPG defaults. +postgresql: + parameters: {} + # max_connections: "200" + # shared_buffers: "256MB" + +# Pod scheduling. NOTE: in CNPG, nodeSelector and tolerations belong *inside* +# affinity - the Cluster spec has no top-level fields for them, and anything +# put there is silently dropped by the API server. +affinity: + enablePodAntiAffinity: true + topologyKey: kubernetes.io/hostname + nodeSelector: {} + tolerations: [] + +# Annotations for the ServiceAccount CNPG creates for the cluster pods. This is +# where IRSA goes when backups write to S3: +# eks.amazonaws.com/role-arn: arn:aws:iam:::role/ +serviceAccount: + annotations: {} + +# Prometheus PodMonitor. Requires the Prometheus Operator CRDs. +monitoring: + enablePodMonitor: false + +# Superuser access is disabled by default (CNPG default too): applications +# should connect as the database owner, not as postgres. +enableSuperuserAccess: false +# Name of a Secret with the superuser credentials. Only used when +# enableSuperuserAccess is true; CNPG generates one if left empty. +superuserSecret: "" + +# UID/GID the postgres process runs as. Empty uses the operator default (26). +postgresUID: "" +postgresGID: "" + +# How replicas are updated: "unsupervised" rolls automatically, +# "supervised" waits for a manual switchover. +primaryUpdateStrategy: unsupervised + +# --------------------------------------------------------------------------- +# Bootstrap: how the database is first created. +# --------------------------------------------------------------------------- +bootstrap: + # Application database and its owner. Both empty uses the CNPG defaults + # ("app"/"app"), which is rarely what you want - name them explicitly. + database: "" + owner: "" + # Name of a Secret with keys `username` and `password` for the owner. Leave + # empty to have CNPG generate the password into Secret/-app. + ownerSecret: "" + # SQL run once, after the database is created. + postInitApplicationSQL: [] + # - CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + + # Logical import from an existing PostgreSQL, for migrating an existing + # database into CNPG. CNPG runs pg_dump/pg_restore against `source` when the + # cluster is created. One-time: it has no effect on an existing cluster. + import: + enabled: false + # microservice = one database per cluster (databases must list exactly one) + type: microservice + databases: [] + source: + host: "" + port: "5432" + dbname: "" + user: postgres + # Secret holding the source user's password + passwordSecret: + name: "" + key: password + +# --------------------------------------------------------------------------- +# Additional databases +# +# bootstrap.initdb creates exactly ONE database. Everything else goes here, as +# CNPG `Database` objects the operator reconciles. +# +# REQUIRES CNPG >= 1.25 (where the Database CRD was introduced). On an older +# operator these objects are ignored silently - no database, and no error. +# +# The owner must be a role that exists. Declare it under managed.roles below, or +# use the cluster's bootstrap owner. +# +# databases: +# - name: keycloak +# owner: keycloak +# extensions: +# - name: uuid-ossp +# +# Each entry supports: name, owner, ensure (present|absent), +# databaseReclaimPolicy (retain|delete), encoding, locale, template, +# allowConnections, connectionLimit, extensions, schemas. +# +# Listing the bootstrap database here fails the render - the operator would be +# asked to create a database that already exists. +# --------------------------------------------------------------------------- +databases: [] + +# --------------------------------------------------------------------------- +# Managed roles +# +# Roles the operator creates and reconciles. Give every application its own role +# owning its own database, so no workload needs superuser rights. +# +# passwordSecret names a Secret with `username` and `password` keys. The operator +# reconciles the role's password to match it. +# +# managed: +# roles: +# - name: keycloak +# passwordSecret: +# name: keycloak-db +# +# A role with neither passwordSecret nor disablePassword fails the render, since +# it would be a login role nobody can authenticate as. +# --------------------------------------------------------------------------- +managed: + roles: [] + +# --------------------------------------------------------------------------- +# Backups: base backups + continuous WAL archiving via the Barman Cloud Plugin. +# +# Disabled by default. Enabling it requires, in this order: +# 1. The Barman Cloud Plugin installed cluster-wide. +# 2. A bucket that the cluster can write to. +# 3. Credentials - on EKS, an IRSA role bound to the cluster ServiceAccount +# via serviceAccount.annotations above. +# --------------------------------------------------------------------------- +backup: + enabled: false + pluginName: barman-cloud.cloudnative-pg.io + + # Must be shorter than any expiry the bucket's own lifecycle policy applies, + # or the bucket deletes backups Barman still counts on. Form: d|w|m. + retentionPolicy: "14d" + + # CNPG uses 6-field cron (sec min hour dom mon dow). Default: daily at 02:00. + schedule: "0 0 2 * * *" + # Take a backup as soon as the ScheduledBackup is created. + immediate: true + backupOwnerReference: self + + objectStore: + # Defaults to -backup + name: "" + # s3 | gcs | azure + provider: s3 + # Bucket name only - the URI scheme comes from provider. + # OAN dev provisions "oan-dev-pg-backup" (see infra-automation). + bucket: "" + # Optional prefix so several clusters can share a bucket, e.g. "/registry" + path: "" + # Full destination path, overriding bucket/path. For S3-compatible stores. + destinationPath: "" + # For S3-compatible object stores (MinIO and similar) + endpointURL: "" + # Pins the folder inside the bucket. Defaults to the cluster name; set it + # explicitly before renaming a cluster that already has backups. + serverName: "" + + # Rendered verbatim as Credentials. The default is IRSA: the + # cluster's ServiceAccount carries the role, no keys anywhere. + # + # S3 with static keys instead (each value is a secret reference): + # credentials: + # accessKeyId: {name: pg-backup-creds, key: ACCESS_KEY_ID} + # secretAccessKey: {name: pg-backup-creds, key: SECRET_ACCESS_KEY} + # GCS on GKE: credentials: {gkeEnvironment: true} + # Azure workload identity: credentials: {inheritFromAzureAD: true} + credentials: + inheritFromIAMRole: true + + wal: + # bzip2 | gzip | lz4 | snappy | xz | zstd + compression: gzip + # AES256 | aws:kms. Empty uses the bucket's default encryption. + encryption: "" + data: + # bzip2 | gzip | lz4 | snappy + compression: gzip + encryption: "" + +# Escape hatch: extra keys merged into the Cluster spec, for CNPG fields this +# chart does not expose yet. Keys the chart already sets cannot be overridden +# here - that would render duplicate YAML keys. +extraClusterSpec: {} + +# Extra labels and annotations on every resource this chart renders +commonLabels: {} +commonAnnotations: {} diff --git a/charts/postgresql-migration/.helmignore b/charts/postgresql-migration/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/postgresql-migration/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/postgresql-migration/CHANGELOG.md b/charts/postgresql-migration/CHANGELOG.md new file mode 100644 index 0000000..05c805d --- /dev/null +++ b/charts/postgresql-migration/CHANGELOG.md @@ -0,0 +1,104 @@ +# Changelog + +All notable changes to the `postgresql-migration` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Removed +- The `00-bootstrap` target and its `CREATE DATABASE` migration. Databases are now + created by the `postgresql-cnpg` chart, via `bootstrap.database` and the CNPG + `Database` CRD, which the operator reconciles properly. Doing it in Flyway meant + `CREATE DATABASE` outside a transaction (with a `.sql.conf` sidecar), no + `IF NOT EXISTS`, and a standing rule never to list the database the cluster + itself creates. Leaving both in place would have failed outright, since the + operator creates the database first. + + This chart now only migrates schemas inside existing databases, which is what + Flyway is for. + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Adapted from an existing Flyway migration chart, rebuilt on +`oan-common` with the previous deployment's specifics removed. + +### Added +- Flyway migration Job, env ConfigMap, generated-script ConfigMap, one ConfigMap + per target, ServiceAccount, and an optional ESO ExternalSecret. +- Multi-target migrations: each entry in `targets` is one database plus its + migration directory, applied in the declared order. `enabled: false` skips an + entry without removing it. +- A bootstrap migration that creates the `keycloak` database, shipping the + `.sql.conf` sidecar with `executeInTransaction=false` that PostgreSQL requires + for `CREATE DATABASE`. +- Runs as a Helm hook (`pre-install`, `pre-upgrade`) by default, so migrations + complete before dependent services start. `hook.enabled: false` installs it as + an ordinary release resource instead. +- `repairOnFailure` (default true) runs `flyway repair` and retries once. +- Render-time validation of `postgresql.host` and + `postgresql.passwordSecret.name`, and of every target: name and database + present, no duplicate names, and a matching directory in the chart. +- Targets with no `.sql` files are skipped at runtime with a log line, so a + database whose schema is managed elsewhere (Keycloak's Liquibase, Sunbird RC's + own DDL) can still be listed for inventory. +- `activeDeadlineSeconds` (default 900), so an unreachable database fails the Job + rather than blocking `helm install` until Helm's timeout. + +### Fixed +- **The database password is no longer written to a ConfigMap.** The original + chart rendered `FLYWAY_PASSWORD` and `PGPASSWORD` into ConfigMap data via its + `configs/env.yaml`, which put a live credential in plaintext in an object that + is not treated as sensitive. It is now injected from a `secretKeyRef`, and the + chart renders no passwords at all. +- Volume mounts for migration directories are now inside the block that emits + the `volumeMounts:` key. Previously the per-folder `range` sat outside the + `if configmap.enabled` guard, so disabling the ConfigMap while keeping + migration folders emitted orphaned list items with no parent key — invalid + YAML. The same applied to `volumes:`. +- `FLYWAY_LOCATIONS` no longer disagrees with the actual mount path. The original + set it to `filesystem:/flyway/migrations` while the script passed + `-locations=filesystem:/migrations/$folder` explicitly and the volumes mounted + at `/migrations/`, so the environment variable was misleading dead + configuration. +- Target ordering is explicit in `targets` rather than derived from + `ls /migrations | sort -n`, which silently depended on directory names carrying + numeric prefixes. + +### Changed +- Built on `oan-common`, replacing the external `common` library chart pulled + from a third-party Helm repository. +- The migration script is generated from `targets` rather than shipped as a + static file that parsed a directory listing, so each target gets its own JDBC + URL and one Job can migrate several databases. +- Script is POSIX `sh`, not `bash`, so it runs on the Alpine-based Flyway image. +- Image tag pinned (`11.10.0-alpine`) rather than defaulting to `latest`. +- Resource requests and limits are now mandatory, via `oan-common.resources`. + +### Removed +- `Service`, `Ingress`, `autoscaling`/HPA, `serviceMonitor`, and liveness and + readiness probes. This chart renders a Job; none of them apply to one. +- `replicaCount`, which a Job does not have. +- Configuration belonging to the previous deployment and unrelated to database + migration: `superset_oauth_clientid`, `superset_oauth_client_secret`, + `kong_ingress_domain`, `gf_auth_generic_oauth_client_id`, + `gf_auth_generic_oauth_client_secret`, `web_console_user`, + `web_console_password`, `web_console_login`, and the `system_settings` block + (`encryption_key`, `default_dataset_id`, `max_event_size`, `dedup_period`). +- The `global.postgresql.password` value, along with `global.*` indirection and + the `base.namespace` override mechanism that let a parent chart retarget the + namespace per subchart. +- The discovery-specific migrations (`02-discover-db`), which belong to a + different service and a different database. diff --git a/charts/postgresql-migration/Chart.yaml b/charts/postgresql-migration/Chart.yaml new file mode 100644 index 0000000..938a0b7 --- /dev/null +++ b/charts/postgresql-migration/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +name: postgresql-migration +description: >- + Flyway schema migrations for OAN PostgreSQL databases, run as a Job. Creates + the per-service databases and applies versioned SQL to each. Runs as a Helm + hook so it completes before the services that depend on it start. +type: application +version: 0.2.0 +# Flyway CLI image tag +appVersion: "11.10.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - postgresql + - flyway + - migration +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://documentation.red-gate.com/flyway +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/postgresql-migration/README.md b/charts/postgresql-migration/README.md new file mode 100644 index 0000000..adf2a78 --- /dev/null +++ b/charts/postgresql-migration/README.md @@ -0,0 +1,160 @@ +# postgresql-migration + +Flyway schema migrations for OAN PostgreSQL databases, run as a Kubernetes Job. + +It does two things, in the order you declare: + +1. **Creates the per-service databases** the cluster does not create itself. +2. **Applies versioned SQL** to each database. + +By default it runs as a Helm hook, so `helm install` waits for migrations to +finish before the services that depend on them start. + +## Install + +```bash +helm dependency update charts/postgresql-migration +helm install migrate charts/postgresql-migration \ + -n oan-registry -f charts/postgresql-migration/examples/registry-stack.dev.yaml +``` + +Run it **after** the database cluster and **before** Keycloak — Keycloak needs the +`keycloak` database to exist. See +[`charts/registry/README.md`](../registry/README.md) for the full stack order. + +## Targets + +A target is one database plus the migration directory applied to it. They run in +the order listed in `targets`: + +```yaml +targets: + - name: 01-registry # directory under files/migrations/ + database: registry # database to connect to (must already exist) + - name: 02-keycloak + database: keycloak +``` + +The list order is what runs; directory names are numbered only so they read in +the same order on disk. A target whose directory holds no `.sql` files is +**skipped**, and the Job logs that it skipped it — so a database can be listed +for inventory purposes without inventing migrations for it. + +Set `enabled: false` on an entry to skip it without deleting it. + +## This chart does not create databases + +Databases are created by the [`postgresql-cnpg`](../postgresql-cnpg) chart — +`bootstrap.database` for the first, and `databases` (CNPG `Database` objects) for +the rest. The operator reconciles them. + +That is deliberate. Creating them here would mean `CREATE DATABASE` outside a +transaction, no `IF NOT EXISTS` to make it idempotent, and a standing rule never +to list the database the cluster already made. The operator does it properly, and +Flyway goes back to what it is for: **migrating schemas inside databases that +already exist.** + +So a target here names a database that must already exist. Pointing one at a +missing database fails with a connection error, not a helpful message. + +## Adding a migration## Adding a migration + +Drop a file into the target's directory, following Flyway's naming +(`V__.sql`): + +``` +charts/postgresql-migration/files/migrations/01-registry/V1__add_lookup_index.sql +``` + +Then bump the chart version and add a CHANGELOG entry. Two rules that Flyway +enforces and this chart cannot soften for you: + +- **Never edit a migration that has already run.** Flyway stores a checksum; + changing the file makes the next run fail validation. Add a new version + instead. +- **Versions must not collide.** Two `V1__` files in the same target is an error. + +Keycloak's own tables are the exception to all of this: it manages its schema +with Liquibase, and a second tool writing those tables would fight it. Do not add +Keycloak table DDL here. + +## The connection + +One host, one role, many databases — each target reuses the connection with a +different database name. + +The role needs: + +- **CREATEDB** for the bootstrap target. +- **Ownership or superuser rights** on each database it migrates. + +In the OAN dev layout that is the CNPG superuser, whose password CNPG generates +into `Secret/-superuser` when the database chart sets +`enableSuperuserAccess: true`. + +`postgresql.host` must be the **primary** (`-rw`). Migrations write, so +a read-only replica service fails — and fails in a way that reads like a +permissions problem. + +## Secrets + +The password is injected as `FLYWAY_PASSWORD` from a `secretKeyRef`. It is +**never** written into a ConfigMap — that was a real flaw in the chart this was +adapted from, where it landed in ConfigMap data in plaintext. + +This chart renders no Secrets; the one it references is created by whatever +manages secrets in that environment. In the OAN dev layout CNPG generates it +(`Secret/-superuser`). + +## Running as a hook, or not + +| | `hook.enabled: true` (default) | `hook.enabled: false` | +|---|---|---| +| When it runs | During `helm install`/`helm upgrade`, before other resources | As an ordinary release resource | +| Helm waits for it | Yes | No | +| `helm uninstall` removes it | No — `deletePolicy` handles cleanup | Yes | +| Repeated upgrades | Works: `before-hook-creation` deletes the old Job first | Fails if the pod template changed — Jobs are immutable | + +`hook.weight: "-5"` orders this ahead of any other hook that needs the schema. +The database cluster must already be running when the hook fires, which holds in +the OAN layout because the cluster is a separate, earlier release. + +## Failure handling + +`repairOnFailure: true` runs `flyway repair` and retries once. Repair rewrites +the schema history to match the migrations on disk, which fixes the common case — +a previous run that failed partway and left a checksum mismatch — but it will +also happily paper over a migration someone edited. Fine for dev; consider +turning it off where you want a failed migration to stay failed until someone +looks at it. + +The Job's `backoffLimit` retries the pod. That is safe: Flyway skips +already-applied migrations. + +`activeDeadlineSeconds` (default 900) stops a Job that cannot reach the database +from blocking `helm install` until Helm's own timeout. + +## Reading what it did + +```bash +kubectl -n oan-registry logs job/migrate-postgresql-migration +``` + +The log names each target, the URL, the migration files it found, and whether it +skipped, succeeded or failed. Per-database history lives in +`flyway_schema_history` in each database. + +Note that with the default hook `deletePolicy: before-hook-creation`, the Job and +its logs survive until the next install or upgrade replaces it. + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema and +[`examples/registry-stack.dev.yaml`](./examples/registry-stack.dev.yaml) for a +per-environment file. + +## Versioning + +Every change — **including adding or editing a migration** — needs a `version` +bump in `Chart.yaml` and an entry in [`CHANGELOG.md`](./CHANGELOG.md). See +[`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/postgresql-migration/ci/lint-values.yaml b/charts/postgresql-migration/ci/lint-values.yaml new file mode 100644 index 0000000..fcf4a69 --- /dev/null +++ b/charts/postgresql-migration/ci/lint-values.yaml @@ -0,0 +1,7 @@ +# Minimum values that let `helm template` run in CI. This chart deliberately +# fails the render when the host and password Secret are unset, so bare defaults +# cannot be templated. Dummy references only - see examples/ for real config. +postgresql: + host: registry-db-rw + passwordSecret: + name: registry-db-superuser diff --git a/charts/postgresql-migration/ci/no-hook-values.yaml b/charts/postgresql-migration/ci/no-hook-values.yaml new file mode 100644 index 0000000..d19c047 --- /dev/null +++ b/charts/postgresql-migration/ci/no-hook-values.yaml @@ -0,0 +1,11 @@ +# Second CI case: the Job as an ordinary release resource rather than a Helm +# hook, which is how you would run migrations on their own. +postgresql: + host: registry-db-rw + passwordSecret: + name: registry-db-superuser +hook: + enabled: false +job: + ttlSecondsAfterFinished: 3600 +repairOnFailure: false diff --git a/charts/postgresql-migration/examples/registry-stack.dev.yaml b/charts/postgresql-migration/examples/registry-stack.dev.yaml new file mode 100644 index 0000000..b5880f2 --- /dev/null +++ b/charts/postgresql-migration/examples/registry-stack.dev.yaml @@ -0,0 +1,55 @@ +# Example: migrations for the registry stack, dev environment. +# +# helm install migrate charts/postgresql-migration \ +# -n oan-registry -f charts/postgresql-migration/examples/registry-stack.dev.yaml +# +# NOTE: with both databases now created by the postgresql-cnpg chart, and with +# Sunbird RC and Keycloak both managing their own schemas, this chart currently +# has nothing to do for the registry stack - every target below is empty and gets +# skipped. It is here for when OAN adds schemas of its own (catalog publish and +# discovery, per engineering-tracker #30), so the targets and their history exist +# from the start rather than being retrofitted. +# +# Run it after the database cluster. + +postgresql: + # The CNPG primary. Migrations write, so this must not be a replica. + host: registry-db-rw + port: 5432 + # Needs rights on each database it migrates. The CNPG superuser has them + # everywhere; a per-database owner would need one release per database. + # + # NOTE: the dev database example leaves enableSuperuserAccess false, since no + # application needs it. Turn it on there before enabling any target that + # actually has SQL, or point this at a per-database owner instead. + user: postgres + passwordSecret: + name: registry-db-superuser + key: password + +targets: + # No .sql files yet: Sunbird RC creates its own tables. The target is kept so + # anything OAN adds alongside the registry has a versioned home. + - name: 01-registry + database: registry + + # No .sql files, and there should not be: Keycloak manages its own schema with + # Liquibase. Listed so the database appears in this chart's inventory. + - name: 02-keycloak + database: keycloak + +# Dev convenience. Consider turning this off in production, so a failed +# migration stays failed until someone reads the log rather than being repaired +# automatically. +repairOnFailure: true + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +commonLabels: + oan.in/environment: dev diff --git a/charts/postgresql-migration/files/migrations/01-registry/README.md b/charts/postgresql-migration/files/migrations/01-registry/README.md new file mode 100644 index 0000000..bd90952 --- /dev/null +++ b/charts/postgresql-migration/files/migrations/01-registry/README.md @@ -0,0 +1,13 @@ +# registry migrations + +Versioned SQL applied to the `registry` database, in Flyway naming order +(`V1__…`, `V2__…`). + +**This directory is intentionally empty of migrations.** Sunbird RC core creates +its own tables from the entity definitions it is given, so the registry needs no +Flyway-managed schema today. The target exists so that anything OAN adds +alongside the registry — a view, an index, a lookup table — has an obvious home +and a versioned history. + +A target whose directory contains no `.sql` files is skipped, and the Job logs +that it skipped it. diff --git a/charts/postgresql-migration/files/migrations/02-keycloak/README.md b/charts/postgresql-migration/files/migrations/02-keycloak/README.md new file mode 100644 index 0000000..8d8f291 --- /dev/null +++ b/charts/postgresql-migration/files/migrations/02-keycloak/README.md @@ -0,0 +1,10 @@ +# keycloak migrations + +Versioned SQL applied to the `keycloak` database. + +**This directory is intentionally empty of migrations.** Keycloak manages its own +schema with Liquibase on startup, and a second migration tool writing the same +tables would fight it. Do not add table DDL for Keycloak's own tables here. + +The target exists only so the database appears in this chart's inventory. A +target whose directory contains no `.sql` files is skipped. diff --git a/charts/postgresql-migration/templates/NOTES.txt b/charts/postgresql-migration/templates/NOTES.txt new file mode 100644 index 0000000..c015735 --- /dev/null +++ b/charts/postgresql-migration/templates/NOTES.txt @@ -0,0 +1,41 @@ +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +{{ if .Values.hook.enabled -}} +The migration Job runs as a Helm hook ({{ join ", " .Values.hook.events }}), so it +has already completed - `helm install` would have failed otherwise. +{{- else -}} +The migration Job is an ordinary release resource, so it is starting now. +{{- end }} + +Targets, in order: +{{- range $targets }} + - {{ .name }} -> database {{ .database }}{{ if not (include "postgresql-migration.targetHasSql" (dict "ctx" $ "name" .name)) }} (no .sql files - will be skipped){{ end }} +{{- end }} + +Connection: {{ .Values.postgresql.user }}@{{ .Values.postgresql.host }}:{{ .Values.postgresql.port }} + +Read what it did: + kubectl -n {{ .Release.Namespace }} logs job/{{ include "postgresql-migration.fullname" . }} + +{{ if .Values.hook.enabled -}} +Because the Job is a hook with deletePolicy "{{ .Values.hook.deletePolicy }}", it +stays until the next install or upgrade replaces it - so the logs above are +readable now, but will be gone after the next run. +{{- end }} + +Check the schema history Flyway keeps, per database: +{{- range $targets }} +{{- if include "postgresql-migration.targetHasSql" (dict "ctx" $ "name" .name) }} + psql -h {{ $.Values.postgresql.host }} -U {{ $.Values.postgresql.user }} -d {{ .database }} \ + -c 'SELECT version, description, success, installed_on FROM flyway_schema_history ORDER BY installed_rank' +{{- end }} +{{- end }} + +If a target failed, the log names which one. The usual causes: + - The role lacks CREATEDB (bootstrap target) or rights on the database. + - A database listed in the bootstrap migration already exists, created by the + PostgreSQL cluster itself. Remove it from the migration - the cluster's + bootstrap database must not be created here. + - A migration was edited after it ran; Flyway rejects the checksum change. + {{ if .Values.repairOnFailure }}repairOnFailure is on, so it retried once after `flyway repair`.{{ else }}repairOnFailure is off, so it failed without retrying.{{ end }} diff --git a/charts/postgresql-migration/templates/_helpers.tpl b/charts/postgresql-migration/templates/_helpers.tpl new file mode 100644 index 0000000..d8bbb21 --- /dev/null +++ b/charts/postgresql-migration/templates/_helpers.tpl @@ -0,0 +1,146 @@ +{{/* +# ============================================================================ +# POSTGRESQL-MIGRATION CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the Flyway target +# and JDBC wiring this chart needs. +# ============================================================================ +*/}} + +{{- define "postgresql-migration.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "postgresql-migration.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "postgresql-migration.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "postgresql-migration.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "postgresql-migration.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "postgresql-migration.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "postgresql-migration.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +Targets that are enabled, in the order given. `enabled` defaults to true when +the key is absent, so a target can be written as just name + database. +*/}} +{{- define "postgresql-migration.enabledTargets" -}} +{{- $out := list -}} +{{- range .Values.targets -}} +{{- if or (not (hasKey . "enabled")) .enabled -}} +{{- $out = append $out . -}} +{{- end -}} +{{- end -}} +{{- $out | toJson -}} +{{- end }} + +{{/* +Validate the connection and every target. Called from the Job and the ConfigMap +so a misconfiguration fails the render rather than the Job. +*/}} +{{- define "postgresql-migration.validate" -}} +{{- $pg := .Values.postgresql -}} +{{- if not $pg.host -}} +{{- fail (printf "%s: postgresql.host is required - point it at the PostgreSQL PRIMARY service, e.g. registry-db-rw. Migrations write, so a read-only replica will not do." .Chart.Name) -}} +{{- end -}} +{{- if not $pg.passwordSecret.name -}} +{{- fail (printf "%s: postgresql.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) -}} +{{- end -}} +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray -}} +{{- if not $targets -}} +{{- fail (printf "%s: no enabled entries in targets - there is nothing to migrate" .Chart.Name) -}} +{{- end -}} +{{- $seen := dict -}} +{{- range $targets -}} +{{- if not .name -}} +{{- fail (printf "%s: every entry in targets needs a name, matching a directory under files/migrations/" $.Chart.Name) -}} +{{- end -}} +{{- if not .database -}} +{{- fail (printf "%s: target %q needs a database" $.Chart.Name .name) -}} +{{- end -}} +{{- if hasKey $seen .name -}} +{{- fail (printf "%s: target %q is listed twice" $.Chart.Name .name) -}} +{{- end -}} +{{- $seen = set $seen .name true -}} +{{- $glob := printf "files/migrations/%s/*" .name -}} +{{- if not ($.Files.Glob $glob) -}} +{{- fail (printf "%s: target %q has no directory files/migrations/%s in the chart" $.Chart.Name .name .name) -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +JDBC URL for one database. +*/}} +{{- define "postgresql-migration.jdbcUrl" -}} +{{- $pg := .ctx.Values.postgresql -}} +{{- printf "jdbc:postgresql://%s:%v/%s%s" $pg.host $pg.port .database $pg.jdbcParams -}} +{{- end }} + +{{/* +Name of the ConfigMap holding the migration files for one target. +*/}} +{{- define "postgresql-migration.targetConfigMapName" -}} +{{- printf "%s-%s" (include "postgresql-migration.fullname" .ctx) .name | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Where a target's migrations are mounted. +*/}} +{{- define "postgresql-migration.targetMountPath" -}} +{{- printf "/migrations/%s" .name -}} +{{- end }} + +{{/* +Migration files for one target, as ConfigMap data. + +Only .sql and .conf are included: the directories also hold READMEs explaining +what belongs in them, and those are not migrations. +*/}} +{{- define "postgresql-migration.targetFiles" -}} +{{- $ctx := .ctx -}} +{{- $name := .name -}} +{{- range $path, $_ := $ctx.Files.Glob (printf "files/migrations/%s/*.sql" $name) }} +{{ base $path }}: |- + {{- $ctx.Files.Get $path | nindent 2 }} +{{- end }} +{{- range $path, $_ := $ctx.Files.Glob (printf "files/migrations/%s/*.conf" $name) }} +{{ base $path }}: |- + {{- $ctx.Files.Get $path | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Does this target actually have migrations to apply? +*/}} +{{- define "postgresql-migration.targetHasSql" -}} +{{- if .ctx.Files.Glob (printf "files/migrations/%s/*.sql" .name) -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +Hook annotations, when the Job runs as a Helm hook. +*/}} +{{- define "postgresql-migration.hookAnnotations" -}} +{{- if .Values.hook.enabled }} +"helm.sh/hook": {{ join "," .Values.hook.events | quote }} +"helm.sh/hook-weight": {{ .Values.hook.weight | quote }} +"helm.sh/hook-delete-policy": {{ .Values.hook.deletePolicy | quote }} +{{- end }} +{{- end }} diff --git a/charts/postgresql-migration/templates/configmap-migrations.yaml b/charts/postgresql-migration/templates/configmap-migrations.yaml new file mode 100644 index 0000000..4bfd143 --- /dev/null +++ b/charts/postgresql-migration/templates/configmap-migrations.yaml @@ -0,0 +1,29 @@ +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +{{- range $targets }} +{{- $files := include "postgresql-migration.targetFiles" (dict "ctx" $ "name" .name) | trim }} +--- +{{/* +Migration files for one target. Rendered even when empty, so the mount exists and +the Job can report "no .sql files" rather than "not mounted". +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-migration.targetConfigMapName" (dict "ctx" $ "name" .name) }} + labels: + {{- include "postgresql-migration.labels" $ | nindent 4 }} + oan.in/migration-target: {{ .name | quote }} + annotations: + {{- include "postgresql-migration.hookAnnotations" $ | nindent 4 }} + {{- with $.Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if $files }} +data: + {{- $files | nindent 2 }} +{{- else }} +# No .sql or .conf files in files/migrations/{{ .name }} - this database's schema +# is managed outside Flyway. The Job skips this target. +data: {} +{{- end }} +{{- end }} diff --git a/charts/postgresql-migration/templates/configmap-script.yaml b/charts/postgresql-migration/templates/configmap-script.yaml new file mode 100644 index 0000000..6e148eb --- /dev/null +++ b/charts/postgresql-migration/templates/configmap-script.yaml @@ -0,0 +1,87 @@ +{{- include "postgresql-migration.validate" . }} +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-migration.fullname" . }}-script + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + migrate.sh: |- + #!/bin/sh + # Generated by the postgresql-migration chart. Do not edit in the cluster - + # change values.yaml and upgrade instead. + # + # POSIX sh, not bash: the flyway alpine image has no bash. + # + # One `flyway migrate` per target, in the order declared in values.yaml. + # Flyway records what it has applied in each database's own + # flyway_schema_history, so re-running this is safe: applied migrations are + # skipped, not repeated. + set -eu + + fail=0 + + run_target() { + name="$1" + url="$2" + dir="/migrations/$name" + + echo "==========================================================" + echo "target: $name" + echo "url: $url" + + if [ ! -d "$dir" ]; then + echo " SKIP: $dir is not mounted" + return 0 + fi + # A target with no .sql is a placeholder for a database whose schema is + # managed elsewhere (Keycloak's Liquibase, Sunbird RC's own DDL). + if [ -z "$(find "$dir" -maxdepth 1 -name '*.sql' -print -quit)" ]; then + echo " SKIP: no .sql files in $dir" + return 0 + fi + + echo " migrations:" + for f in "$dir"/*.sql; do + echo " $(basename "$f")" + done + + if flyway -url="$url" -locations="filesystem:$dir" migrate; then + echo " OK: $name" + return 0 + fi + + {{- if .Values.repairOnFailure }} + echo " migrate failed for $name - running repair and retrying once" + if ! flyway -url="$url" -locations="filesystem:$dir" repair; then + echo " FAILED: repair failed for $name" + return 1 + fi + if flyway -url="$url" -locations="filesystem:$dir" migrate; then + echo " OK: $name (after repair)" + return 0 + fi + echo " FAILED: $name still failing after repair" + return 1 + {{- else }} + echo " FAILED: $name (repairOnFailure is disabled, so not retrying)" + return 1 + {{- end }} + } + + {{- range $targets }} + run_target {{ .name | quote }} {{ include "postgresql-migration.jdbcUrl" (dict "ctx" $ "database" .database) | quote }} || fail=1 + {{- end }} + + echo "==========================================================" + if [ "$fail" -ne 0 ]; then + echo "one or more targets FAILED" + exit 1 + fi + echo "all targets completed" diff --git a/charts/postgresql-migration/templates/configmap.yaml b/charts/postgresql-migration/templates/configmap.yaml new file mode 100644 index 0000000..c50910b --- /dev/null +++ b/charts/postgresql-migration/templates/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-migration.envConfigMapName" . }} + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + # Flyway settings. The password is NOT here - it comes from a Secret, injected + # as FLYWAY_PASSWORD in the Job. + FLYWAY_USER: {{ .Values.postgresql.user | quote }} + FLYWAY_BASELINE_ON_MIGRATE: {{ .Values.flyway.baselineOnMigrate | quote }} + FLYWAY_BASELINE_VERSION: {{ .Values.flyway.baselineVersion | quote }} + FLYWAY_VALIDATE_ON_MIGRATE: {{ .Values.flyway.validateOnMigrate | quote }} + FLYWAY_OUT_OF_ORDER: {{ .Values.flyway.outOfOrder | quote }} + {{- range $k, $v := .Values.flyway.extraEnv }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- range $k, $v := .Values.envConfig }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/charts/postgresql-migration/templates/job.yaml b/charts/postgresql-migration/templates/job.yaml new file mode 100644 index 0000000..585b954 --- /dev/null +++ b/charts/postgresql-migration/templates/job.yaml @@ -0,0 +1,105 @@ +{{- include "postgresql-migration.validate" . }} +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "postgresql-migration.fullname" . }} + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + backoffLimit: {{ .Values.job.backoffLimit }} + {{- with .Values.job.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ . }} + {{- end }} + {{- with .Values.job.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ . }} + {{- end }} + template: + metadata: + labels: + {{- include "postgresql-migration.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: {{ .Values.job.restartPolicy }} + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "postgresql-migration.serviceAccountName" . }} + {{- end }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + containers: + - name: flyway + image: {{ include "postgresql-migration.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + command: + - /bin/sh + - /script/migrate.sh + env: + - name: FLYWAY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.passwordSecret.name }} + key: {{ .Values.postgresql.passwordSecret.key }} + {{- with (include "oan-common.env" . | trim) }} + {{- . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ include "postgresql-migration.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + - name: script + mountPath: /script + readOnly: true + {{- range $targets }} + - name: {{ printf "migrations-%s" .name | trunc 63 | trimSuffix "-" }} + mountPath: {{ include "postgresql-migration.targetMountPath" (dict "name" .name) }} + readOnly: true + {{- end }} + volumes: + - name: script + configMap: + name: {{ include "postgresql-migration.fullname" . }}-script + defaultMode: 0555 + {{- range $targets }} + - name: {{ printf "migrations-%s" .name | trunc 63 | trimSuffix "-" }} + configMap: + name: {{ include "postgresql-migration.targetConfigMapName" (dict "ctx" $ "name" .name) }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/postgresql-migration/templates/serviceaccount.yaml b/charts/postgresql-migration/templates/serviceaccount.yaml new file mode 100644 index 0000000..0097ee0 --- /dev/null +++ b/charts/postgresql-migration/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "postgresql-migration.serviceAccountName" . }} + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/postgresql-migration/values.yaml b/charts/postgresql-migration/values.yaml new file mode 100644 index 0000000..d54e215 --- /dev/null +++ b/charts/postgresql-migration/values.yaml @@ -0,0 +1,194 @@ +# ============================================================================ +# postgresql-migration - default values +# +# Flyway migrations for OAN PostgreSQL databases, run as a Job. It does two +# things, in order: +# +# 1. Creates the per-service databases the cluster does not create itself. +# 2. Applies versioned SQL to each database. +# +# It runs as a Helm hook by default, so `helm install` of a service chart waits +# for migrations to finish before the service starts. +# +# Minimum you must set: +# - postgresql.host +# - postgresql.passwordSecret.name +# ============================================================================ + +image: + registry: docker.io + repository: flyway/flyway + # Pinned; `latest` in a migration job means a tool upgrade you did not choose. + tag: "11.10.0-alpine" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + # The Job talks to PostgreSQL, not the Kubernetes API. + automountServiceAccountToken: false + +# --------------------------------------------------------------------------- +# Connection +# +# One host, one role, many databases: each target below reuses this connection +# with a different database name. +# +# The role needs ownership or superuser rights on each database it migrates, in +# order to create objects and the flyway_schema_history table. +# +# This chart does NOT create databases. That is the postgresql-cnpg chart's job - +# `bootstrap.database` for the first one and `databases` for the rest, both +# reconciled by the CNPG operator. Creating them here instead would mean +# CREATE DATABASE outside a transaction, no IF NOT EXISTS, and a rule about never +# listing the database the cluster already made. The operator does it properly. +# --------------------------------------------------------------------------- +postgresql: + # PostgreSQL primary service, e.g. "registry-db-rw". Must be the primary: + # migrations write. + host: "" + port: 5432 + user: postgres + # Secret holding the password. Required - the chart renders no passwords. + passwordSecret: + name: "" + key: password + # Appended to every JDBC URL, e.g. "?sslmode=require" + jdbcParams: "" + +# --------------------------------------------------------------------------- +# Targets +# +# Each entry is one database and the migration directory applied to it, run in +# the order listed. The order in this list is what runs - directory names are +# numbered only so they read in the same order on disk. +# +# name: the directory under files/migrations/ +# database: the database to connect to +# enabled: set false to skip without removing the entry +# +# A target whose directory has no .sql files is skipped, and the Job says so. +# --------------------------------------------------------------------------- +targets: + - name: 01-registry + database: registry + enabled: true + - name: 02-keycloak + database: keycloak + enabled: true + +# Run `flyway repair` and retry once when a migration fails. +# +# Repair rewrites the schema history to match the migrations on disk. It fixes +# the common case - a previous run that failed partway and left a checksum +# mismatch - but it will also happily paper over a genuinely edited migration. +# Leave it on for dev; consider turning it off where you want a failed migration +# to stay failed until someone looks at it. +repairOnFailure: true + +# Flyway's own settings, passed through as environment variables. Anything +# Flyway reads from FLYWAY_* can go here. +flyway: + # Create the schema history table on a database that has objects but no + # history yet, instead of refusing to run. + baselineOnMigrate: true + # Version recorded for that baseline. + baselineVersion: "0" + # Reject a migration whose checksum no longer matches the recorded one. + validateOnMigrate: true + # Allow out-of-order migrations. Off: a lower version arriving late is an error. + outOfOrder: false + # Extra FLYWAY_* environment variables, as name -> value. + extraEnv: {} + # FLYWAY_CONNECT_RETRIES: "10" + +# --------------------------------------------------------------------------- +# Helm hook +# +# As a hook, the Job runs during `helm install`/`helm upgrade` and Helm waits for +# it before proceeding. That is what orders migrations ahead of the services. +# +# Two consequences worth knowing: +# - A hook resource is not tracked in the release, so `helm uninstall` does not +# remove it; deletePolicy handles the cleanup instead. +# - The database cluster must already be running when the hook fires. That +# holds in the OAN layout, where the cluster is a separate, earlier release. +# +# Set enabled: false to install the Job as an ordinary release resource, e.g. to +# run migrations by themselves. +# --------------------------------------------------------------------------- +hook: + enabled: true + events: + - pre-install + - pre-upgrade + # Lower weights run first, so keep this below any other hook that needs the + # schema to exist. + weight: "-5" + # before-hook-creation deletes the previous Job before creating this one, + # which is what makes repeated upgrades work: a Job's pod template is + # immutable, so re-applying a changed one is rejected. + deletePolicy: before-hook-creation + +job: + # Pod-level retries. Flyway is idempotent - already-applied migrations are + # skipped - so a retry is safe. + backoffLimit: 3 + # Give up after this long. Without it a Job that cannot reach the database + # blocks `helm install` until Helm's own timeout. + activeDeadlineSeconds: 900 + # Delete the finished Job (and its pod, and its logs) this long after it ends. + # Empty keeps it until the next run, so the logs are there to read. + ttlSecondsAfterFinished: "" + restartPolicy: OnFailure + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +# Off by default, as elsewhere in this repo. Flyway only reads its migrations and +# writes to the database, so it is a good candidate for tightening - but verify +# the pinned image's own user before enabling runAsUser, since a mismatch shows +# up as a permission error on /flyway rather than anything obvious. +podSecurityContext: + enabled: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# Non-secret extra configuration, injected via envFrom. +envConfig: {} + +# Extra env vars from specific Secret keys. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} From 363003b3d9dd1ab5cb3a7e3077f3b62b69d74112 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:35:04 +0530 Subject: [PATCH 15/81] feat: add keycloak chart with sunbird-rc realm import [#53] Imports the sunbird-rc realm from a ConfigMap on first start, with the realm content checksummed into the pod annotations so a realm change rolls the pod. An init container waits for the database, standing in for compose's depends_on: condition: service_healthy. Keycloak connects as the owner of its own keycloak database rather than sharing the registry's database as postgres, which is how the compose stack runs it. The realm JSON is Sunbird RC's export and still carries its upstream defaults: a placeholder admin-api client secret, an enabled placeholder user with a known password, and a wildcard redirect URI on the public frontend client. All three need hardening before this reaches any environment that is not local. See the chart README for the client secret step; the other two are tracked as follow-up. --- charts/keycloak/.helmignore | 13 + charts/keycloak/CHANGELOG.md | 72 + charts/keycloak/Chart.yaml | 28 + charts/keycloak/README.md | 194 ++ charts/keycloak/ci/lint-values.yaml | 12 + charts/keycloak/examples/keycloak.dev.yaml | 77 + charts/keycloak/examples/keycloak.prod.yaml | 97 + charts/keycloak/files/realm-export.json | 2312 +++++++++++++++++ charts/keycloak/templates/NOTES.txt | 36 + charts/keycloak/templates/_helpers.tpl | 159 ++ .../keycloak/templates/configmap-realm.yaml | 27 + charts/keycloak/templates/configmap.yaml | 16 + charts/keycloak/templates/deployment.yaml | 126 + charts/keycloak/templates/ingress.yaml | 41 + .../templates/poddisruptionbudget.yaml | 33 + charts/keycloak/templates/service.yaml | 19 + charts/keycloak/templates/serviceaccount.yaml | 13 + .../templates/tests/test-connection.yaml | 39 + charts/keycloak/values.yaml | 313 +++ 19 files changed, 3627 insertions(+) create mode 100644 charts/keycloak/.helmignore create mode 100644 charts/keycloak/CHANGELOG.md create mode 100644 charts/keycloak/Chart.yaml create mode 100644 charts/keycloak/README.md create mode 100644 charts/keycloak/ci/lint-values.yaml create mode 100644 charts/keycloak/examples/keycloak.dev.yaml create mode 100644 charts/keycloak/examples/keycloak.prod.yaml create mode 100644 charts/keycloak/files/realm-export.json create mode 100644 charts/keycloak/templates/NOTES.txt create mode 100644 charts/keycloak/templates/_helpers.tpl create mode 100644 charts/keycloak/templates/configmap-realm.yaml create mode 100644 charts/keycloak/templates/configmap.yaml create mode 100644 charts/keycloak/templates/deployment.yaml create mode 100644 charts/keycloak/templates/ingress.yaml create mode 100644 charts/keycloak/templates/poddisruptionbudget.yaml create mode 100644 charts/keycloak/templates/service.yaml create mode 100644 charts/keycloak/templates/serviceaccount.yaml create mode 100644 charts/keycloak/templates/tests/test-connection.yaml create mode 100644 charts/keycloak/values.yaml diff --git a/charts/keycloak/.helmignore b/charts/keycloak/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/keycloak/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/keycloak/CHANGELOG.md b/charts/keycloak/CHANGELOG.md new file mode 100644 index 0000000..3977df8 --- /dev/null +++ b/charts/keycloak/CHANGELOG.md @@ -0,0 +1,72 @@ +# Changelog + +All notable changes to the `keycloak` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Configuration ported from the verified `registry/docker-compose.yml` +stack, with the deployment shape taken from Sunbird's own Helm charts +(`deploy-as-code/helm/v2`). + +### Added +- Deployment, Service, env ConfigMap, realm ConfigMap, ServiceAccount, and + optional Ingress and ESO ExternalSecret, built on the `oan-common` library. +- All ten Keycloak environment variables from the compose stack, verified at + parity: `DB_VENDOR`, `DB_ADDR`, `DB_PORT`, `DB_DATABASE`, `DB_USER`, + `DB_PASSWORD`, `KEYCLOAK_USER`, `KEYCLOAK_PASSWORD`, `KEYCLOAK_IMPORT` and + `PROXY_ADDRESS_FORWARDING`. +- The `sunbird-rc` realm export shipped in `files/`, rendered into a ConfigMap + and mounted for `KEYCLOAK_IMPORT`, so `helm install` needs nothing prepared + beforehand. A `checksum/realm` annotation rolls the pod when it changes. +- `realmImport.existingConfigMap` to mount a realm ConfigMap managed outside the + chart instead, and `realmImport.realmJson` to supply one inline. +- Render-time validation that `database.host`, `database.passwordSecret.name` + and `admin.passwordSecret.name` are set, since the chart creates no passwords. +- `strategy: Recreate`, so two instances never run the realm import at once. + +- Init containers that wait for the database before starting, so a first install + does not crashloop while PostgreSQL comes up. Derived from `database.host`. +- Optional PodDisruptionBudget, which fails the render if enabled alongside a + single replica - that combination blocks node drains entirely. +- `helm test` check that `/auth` responds and the imported realm is served. +- `extraVolumes` / `extraVolumeMounts` / `extraInitContainers` escape hatches. +- Per-environment example values for dev and production. The production example + connects as a dedicated `keycloak` role rather than the superuser, pulls the + admin password from ESO, and documents why it stays at one replica: the legacy + WildFly distribution needs Infinispan cache clustering configured before a + second replica can share sessions. + +### Notes +- The shipped realm is a second copy of the compose stack's + `registry/imports/realm-export.json`, byte-identical at the time of writing. + Nothing enforces that: if the compose copy changes, this one needs updating and + a chart version bump. +- Editing the realm rolls the pod but does not re-import it. The legacy image + imports a realm only when it is absent, so changes to a live realm have to be + made in the Keycloak console or against a fresh database. + +### Changed from the compose stack +- Probes follow Sunbird's Helm chart, not the compose healthcheck: readiness and + startup check `/auth/` on 8080, liveness is a TCP check. Compose curls the + WildFly management port 9990, which this chart deliberately does not expose. +- Image tag defaults to `v1.0.0` rather than `latest`, matching the version + Sunbird pairs with `sunbird-rc-core:v2.0.0`. A chart should not deploy a + moving tag. +- Resource limits added. Sunbird's chart sets requests only; limits are + mandatory in this repo. diff --git a/charts/keycloak/Chart.yaml b/charts/keycloak/Chart.yaml new file mode 100644 index 0000000..d93c746 --- /dev/null +++ b/charts/keycloak/Chart.yaml @@ -0,0 +1,28 @@ +apiVersion: v2 +name: keycloak +description: >- + Keycloak for the OAN registry, on the Sunbird RC Keycloak image. Issues and + validates the tokens the registry authorises requests with, and imports the + realm, clients and roles the registry expects. +type: application +version: 0.2.0 +# Tag of ghcr.io/sunbird-rc/sunbird-rc-keycloak. Sunbird's own v2 deployment +# pairs this version with sunbird-rc-core v2.0.0. +appVersion: "v1.0.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - keycloak + - sunbird-rc + - authentication +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/Sunbird-RC/devops/tree/main/deploy-as-code +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/keycloak/README.md b/charts/keycloak/README.md new file mode 100644 index 0000000..d69576e --- /dev/null +++ b/charts/keycloak/README.md @@ -0,0 +1,194 @@ +# keycloak + +Keycloak for the OAN registry, on the [Sunbird RC Keycloak +image](https://github.com/Sunbird-RC/devops/tree/main/deploy-as-code). It issues +and validates the tokens the registry authorises requests with, and imports the +realm, clients and roles the registry expects. + +This is the **legacy (WildFly/JBoss) Keycloak distribution**: it serves under +`/auth` and takes `DB_VENDOR`-style environment variables, not the modern `KC_*` +ones. Charts written for upstream Keycloak will not work with this image, which +is why this chart exists rather than depending on one. + +## What it renders + +| Resource | Notes | +|---|---| +| Deployment | `strategy: Recreate` — one instance, and the realm import runs on startup. Init container waits for the database | +| Service | `ClusterIP` on 8080 | +| ConfigMap (env) | Extra non-secret config from `envConfig` | +| ConfigMap (realm) | The realm shipped in `files/`, mounted for `KEYCLOAK_IMPORT` | +| ServiceAccount | | +| PodDisruptionBudget | Optional, off by default | +| Ingress | Optional, off by default | +| Test Pod | `helm test` check on `/auth` and the imported realm | + +Configuration matches `registry/docker-compose.yml` exactly — all ten Keycloak +environment variables, same names and same values. + +## Install + +```bash +helm dependency update charts/keycloak +helm install keycloak charts/keycloak \ + -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml +``` + +Requires a reachable PostgreSQL — see [`postgresql-cnpg`](../postgresql-cnpg) — +and two Secrets that this chart never creates: the database password and the +Keycloak admin password. + +## The two-phase first install + +**The registry cannot authenticate immediately after this chart installs.** The +realm export ships with the `admin-api` client secret masked (`**********`), +which is not a working credential. So: + +1. Install this chart; it imports the realm on first start. +2. Reach the admin console: + ```bash + kubectl -n oan-registry port-forward svc/keycloak 8080:8080 + ``` + then open `http://localhost:8080/auth/admin`. +3. Realm `sunbird-rc` → Clients → `admin-api` → Credentials → **Regenerate + Secret**. +4. Store that value where the registry chart's + `keycloak.adminClientSecret` points. +5. Install the registry. + +There is no way for a chart to shortcut this while the realm is imported from a +masked export. Making it one-phase means managing clients declaratively instead +— `keycloak-config-cli` or the Keycloak Operator — which is a larger change. + +## Waiting for the database + +Kubernetes has no equivalent of compose's `depends_on: condition: +service_healthy`, so an init container waits for the database before Keycloak +starts: + +```yaml +waitFor: + enabled: true + database: true # host and port come from `database` +``` + +Without it, a first install crashloops while PostgreSQL comes up — which recovers +on its own, but looks broken. + +## Verifying an install + +```bash +helm test keycloak -n oan-registry +``` + +Checks that `/auth` responds and, when `realmImport.enabled`, that the realm is +actually being served. That second check is the one that matters: the registry +validates tokens against that exact realm URL. + +## Replicas and clustering + +**This chart stays at one replica, and that is deliberate.** The legacy WildFly +distribution needs its Infinispan cache cluster configured — JGroups discovery +and session replication — before a second replica is safe. Without it, a token +issued by one replica is not recognised by the other. This chart does not +configure that, so `replicaCount: 1` and `strategy: Recreate` are the honest +settings. + +The practical consequence: restarting Keycloak briefly interrupts token issuing. +The registry keeps serving requests whose tokens are already valid. + +Getting past this means either configuring Infinispan clustering, or moving to a +current Keycloak with the Keycloak Operator — which would also remove the masked +client-secret problem below. + +`podDisruptionBudget` is therefore off by default: over a single pod it blocks +node drains entirely. + +## Per-environment values + +| File | For | +|---|---| +| [`examples/keycloak.dev.yaml`](./examples/keycloak.dev.yaml) | Dev: superuser connection, hand-made Secrets, no ingress | +| [`examples/keycloak.prod.yaml`](./examples/keycloak.prod.yaml) | Production: dedicated `keycloak` role, JVM heap sized to the limit | + +## Realm management + +The chart ships `files/realm-export.json` — realm `sunbird-rc`, 8 clients, +including the `admin-api` and `registry-frontend` clients the registry uses — and +renders it into a ConfigMap mounted at `realmImport.mountPath`. The legacy image +imports it on startup via `KEYCLOAK_IMPORT`. + +That means **`helm install` needs nothing prepared beforehand**, and a realm +change rolls the pod through the `checksum/realm` annotation. + +| You want | Set | +|---|---| +| The shipped realm | nothing — this is the default | +| A realm managed outside the chart | `existingConfigMap: ` (must already exist; the chart renders none) | +| A different realm inline | `realmJson: ` | +| No import at all | `enabled: false` | + +### Two things to know + +**The shipped realm is a second copy.** It is byte-identical to the compose +stack's `registry/imports/realm-export.json`, which is deliberate — the cluster +should import the same realm that was verified locally. But nothing enforces +that: if the compose copy changes, this one has to be updated too, with a chart +version bump. That is the cost of the chart being self-contained. + +**Editing the realm does not re-import it.** The legacy image imports a realm +only when it is *absent*. Changing the JSON rolls the pod, but Keycloak will +leave an already-imported realm alone — so changes to a live realm have to be +made in the Keycloak console, or by starting from a fresh database. This catches +people out, because the pod restart makes it look like something happened. + +## Database## Database + +The chart's `values.yaml` defaults match compose — Keycloak sharing the +registry's database — but **the example values point it at its own `keycloak` +database**, which is how the stack is meant to be deployed. Sunbird's own Helm +charts do the same. Keycloak's realm tables have no reason to sit beside registry +records, and separating them frees the registry to connect as a non-superuser +owner. + +The `keycloak` database is created by +[`postgresql-migration`](../postgresql-migration)'s bootstrap target, which must +run before this chart. To reproduce compose exactly instead, set +`database.name: registry` and drop the migration step. + +## Probes + +These follow Sunbird's own chart rather than the compose healthcheck. Compose +curls the WildFly management port (9990); this chart does not expose 9990, +because nothing in-cluster needs it and the admin console is on 8080 under +`/auth/admin`. Readiness instead checks that Keycloak is really serving `/auth`. + +First start imports the realm and migrates the schema, so `startupProbe` carries +the slow path (up to 5 minutes by default) and liveness stays tight. + +## Image tag + +Compose uses `latest`. A chart must not deploy a moving tag, so this defaults to +`v1.0.0` — the version Sunbird's own v2 deployment pairs with +`sunbird-rc-core:v2.0.0`. Set `image.digest` to pin exactly. + +## Ingress and the issuer URL + +`proxyAddressForwarding` is on, matching compose, so Keycloak will honour +`X-Forwarded-Host` / `X-Forwarded-Proto`. If you enable ingress: + +- The controller must actually set those headers. +- The registry's `keycloak.url` must be whichever URL Keycloak puts in the `iss` + claim. A mismatch rejects every authenticated request, and the failure reads + like a permissions problem rather than a configuration one. + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema, and +[`examples/keycloak.dev.yaml`](./examples/keycloak.dev.yaml) for a per-environment +file. + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/keycloak/ci/lint-values.yaml b/charts/keycloak/ci/lint-values.yaml new file mode 100644 index 0000000..85da7a6 --- /dev/null +++ b/charts/keycloak/ci/lint-values.yaml @@ -0,0 +1,12 @@ +# Minimum values that let `helm template` run in CI. +# +# This chart deliberately fails the render when the database and password +# Secrets are unset, so bare defaults cannot be templated. These are dummy +# references, not real configuration - see examples/ for that. +database: + host: registry-db-rw + passwordSecret: + name: registry-db-app +admin: + passwordSecret: + name: keycloak-admin diff --git a/charts/keycloak/examples/keycloak.dev.yaml b/charts/keycloak/examples/keycloak.dev.yaml new file mode 100644 index 0000000..fbc9cc3 --- /dev/null +++ b/charts/keycloak/examples/keycloak.dev.yaml @@ -0,0 +1,77 @@ +# Example: Keycloak for the registry, dev environment. +# +# helm install keycloak charts/keycloak \ +# -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml +# +# Install order: postgresql-cnpg -> THIS -> regenerate admin-api secret -> +# registry. See charts/registry/README.md. + +# No fullnameOverride needed: the chart is named "keycloak", so a release named +# "keycloak" already produces Service/keycloak. Set it only if you install under +# a different release name and still want the service to be reachable as +# "keycloak" - the registry validates the token issuer against this host. + +# Keycloak gets its OWN database and its OWN role, both created by the +# postgresql-cnpg chart (`databases` and `managed.roles`). This is where the +# cluster deployment deliberately improves on compose, which shares one +# `registry` database between both services as the postgres superuser: Keycloak's +# realm tables have no reason to sit beside registry records, and nothing here +# needs superuser rights. +# +# To reproduce compose exactly instead, set name: registry, user: postgres, and +# point passwordSecret at registry-db-superuser. +database: + host: registry-db-rw + port: 5432 + name: keycloak + user: keycloak + # The same Secret the managed role is reconciled against, so the password the + # role has and the password Keycloak sends cannot drift. + passwordSecret: + name: keycloak-db + key: password + +admin: + username: admin + # Create by hand until ESO is available: + # kubectl -n oan-registry create secret generic keycloak-admin \ + # --from-literal=keycloakAdminPassword='' + passwordSecret: + name: keycloak-admin + key: keycloakAdminPassword + +# Ships the realm the registry expects. Its admin-api client secret is masked +# and must be regenerated in the console after first start. +# Uses the realm shipped with the chart (files/realm-export.json), so nothing +# needs preparing before install. +realmImport: + enabled: true + +# Dev: reach it with `kubectl port-forward` rather than an ingress. Enabling +# ingress means the controller must set X-Forwarded-Host / X-Forwarded-Proto, +# and the registry's keycloak.url must then be the external URL. +ingress: + enabled: false + +resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +commonLabels: + oan.in/environment: dev + +# Blocks startup until the database accepts connections, standing in for +# compose's `depends_on: condition: service_healthy`. Host and port are taken +# from `database` above. +waitFor: + enabled: true + database: true + +# `helm test keycloak -n oan-registry` checks /auth and the imported realm. +tests: + enabled: true + realm: sunbird-rc diff --git a/charts/keycloak/examples/keycloak.prod.yaml b/charts/keycloak/examples/keycloak.prod.yaml new file mode 100644 index 0000000..e64d40b --- /dev/null +++ b/charts/keycloak/examples/keycloak.prod.yaml @@ -0,0 +1,97 @@ +# Example: Keycloak for the registry, production. +# +# helm install keycloak charts/keycloak \ +# -n oan-registry -f charts/keycloak/examples/keycloak.prod.yaml + +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-keycloak + tag: "v1.0.0" + # digest: "sha256:..." # pin this before going live + +# NOTE: this is the legacy WildFly Keycloak distribution. Running two replicas +# needs its Infinispan cache cluster configured (JGroups discovery, session +# replication) or the two will not share sessions - a token issued by one is not +# recognised by the other. That is not configured by this chart, so production +# stays at one replica until either that is set up or the deployment moves to a +# current Keycloak with the Keycloak Operator. +# +# The practical consequence: a Keycloak restart briefly interrupts token issuing. +# The registry keeps serving requests whose tokens are already valid. +replicaCount: 1 + +database: + host: registry-db-rw + port: 5432 + name: keycloak + # Production uses a dedicated role owning only the keycloak database, not the + # superuser. Requires that role and database to exist - see + # postgresql-migration. + user: keycloak + passwordSecret: + name: keycloak-db + key: password + +admin: + username: admin + passwordSecret: + name: keycloak-admin + key: keycloakAdminPassword + +# The admin password Secret is created out of band, by whatever manages secrets +# in this environment. This chart never renders a password. +# kubectl -n oan-registry create secret generic keycloak-admin \ +# --from-literal=keycloakAdminPassword='' + +# Uses the realm shipped with the chart (files/realm-export.json), so nothing +# needs preparing before install. +realmImport: + enabled: true + +# Reachable from outside only if users need the login page. Keep it internal +# otherwise - the registry reaches Keycloak over cluster DNS. +# +# If you do expose it: proxyAddressForwarding below is already on, the controller +# must set X-Forwarded-Host and X-Forwarded-Proto, and the registry's +# keycloak.url must then be the EXTERNAL URL, because that is what Keycloak will +# put in the token issuer. +ingress: + enabled: false + # className: alb + # annotations: + # alb.ingress.kubernetes.io/scheme: internal + # hosts: + # - host: keycloak.oan.example + # paths: + # - path: /auth + # pathType: Prefix + # tls: + # - hosts: [keycloak.oan.example] + # secretName: keycloak-tls + +proxyAddressForwarding: true + +resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 3Gi + +# Not enabled: with replicaCount 1 a PDB blocks node drains entirely. Revisit +# together with the clustering note above. +podDisruptionBudget: + enabled: false + +podSecurityContext: + enabled: false +securityContext: + enabled: false + +envConfig: + # Keep the JVM heap inside the memory limit above. + JAVA_OPTS_APPEND: "-Xms1024m -Xmx2048m" + +commonLabels: + oan.in/environment: prod diff --git a/charts/keycloak/files/realm-export.json b/charts/keycloak/files/realm-export.json new file mode 100644 index 0000000..4b343c1 --- /dev/null +++ b/charts/keycloak/files/realm-export.json @@ -0,0 +1,2312 @@ +{ + "id": "sunbird-rc", + "realm": "sunbird-rc", + "displayName": "Sunbird Rc Core", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "a772a1cd-7904-4e5c-a864-5041fa69d491", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "42dba8cf-f483-4668-a087-cba46ed86ad2", + "name": "admin", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "5fa4077d-1686-4506-97a6-5bce1bce59dc", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "network_operator", + "description": "Network Operator - onboards and governs Providers", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "270ecc82-3249-475c-a851-d3ea162059b8", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "0de39ec0-7602-4aa2-b54d-ab12e9bdb76f", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4259031b-736e-49eb-9e70-4a312a48e211", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9887e071-49b0-464b-b6fe-a1c585a709c7", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7d61f967-1dce-482f-96e5-9eff79eb4851", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-events", + "manage-identity-providers", + "manage-clients", + "view-identity-providers", + "manage-authorization", + "view-users", + "manage-users", + "manage-events", + "manage-realm", + "impersonation", + "view-authorization", + "query-clients", + "create-client", + "view-clients", + "query-users", + "query-realms", + "view-realm", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "e93b1761-fb32-46c5-bfa2-4b853c7b5573", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "bc32a137-07a1-40f0-b9fd-a6e64e27f99b", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4b5abd90-d6a2-4981-a50f-520292496f0b", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "99d2ed5f-00a9-44ed-8b9f-bdd7ba3facb8", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "3fbd2cd5-0698-490e-a52f-ef528d001a62", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9b8b4f1c-5ed6-49ca-bec3-0a9a4867ad26", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7e6341ff-a1d8-4400-af94-3a007a06706a", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ad0c87da-9f34-4992-a83a-f6b924f1944d", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "badb0d4d-06da-45e8-a777-ef47f712d3ed", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "f8f48f0f-bd2a-4cb7-9b77-af69b9805c25", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ca0b1e94-6578-4295-abf4-ae99f7df7595", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ff7230eb-7dae-44a5-8f68-f68747f35589", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "890d054b-86f9-49f5-8dd9-14f62aa956de", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "627f3f4c-58e3-49f3-9989-a05d4d0a8752", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-api": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "2e9bfeff-129e-4072-9617-5847644aac24", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "attributes": {} + } + ], + "account": [ + { + "id": "5694c2d0-6d02-4182-bb09-78f4f5f1ec2d", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0612622f-dae7-48f8-8985-fe7e5ab8acc7", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "eeefbd57-94b8-4d7d-bf2f-075c39ccb746", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "9e9165b9-1170-47ab-802a-aecffefb3ab7", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "a8d0a100-e382-49ba-ac42-48dbf815a2de", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "08772792-146d-4676-ba2d-ce56b0104263", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0a2e7893-784e-47ef-ba35-4a26901350c0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + } + ], + "registry-frontend": [] + } + }, + "groups": [], + "defaultRole": { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "sunbird-rc" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpSupportedApplications": [ + "FreeOTP", + "Google Authenticator" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "users": [ + { + "id": "3cc9ac60-b67d-4c57-8005-acd4d236b2dc", + "createdTimestamp": 1634296700339, + "username": "service-account-admin-api", + "enabled": true, + "totp": false, + "emailVerified": false, + "serviceAccountClientId": "admin-api", + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-sunbird-rc", + "admin" + ], + "clientRoles": { + "realm-management": [ + "manage-users", + "manage-realm" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "username": "no-user", + "enabled": true, + "emailVerified": false, + "credentials": [ + { + "type": "password", + "value": "no-user-password", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-sunbird-rc", + "network_operator" + ] + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account" + ] + } + ] + }, + "clients": [ + { + "id": "01326f76-7838-47fc-ae62-399a75c5ae38", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "f871d6fc-d997-4ac6-99fe-d797955bc9f0", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "8ab32c51-9aa0-4e28-80bf-0d6b53151354", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "25962708-6d45-47d9-8935-5db159234aac", + "clientId": "admin-api", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "*", + "http://localhost:4200/", + "http://localhost:4200/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "84ae9d6c-424f-47f0-9d4d-f2e98fed7339", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "98406938-b8db-4992-8519-917054f6ed0e", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "id": "90d6b17a-5a06-4546-8091-960301f8147e", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b245b10b-606c-417c-bbc0-8f81a7a992a6", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "961a8a92-1598-48ff-adee-1e5fee0df757", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "69c04ae8-6669-48e7-8234-08986a7f490d", + "clientId": "registry-frontend", + "name": "Registry Frontend", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "login_theme": "sunbird-rc", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b777b14f-b0e8-4da5-a802-092803319cbe", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/sunbird-rc/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/sunbird-rc/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "7160f35d-97d3-4730-9769-4b03b32e5191", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "b4695333-f842-4ef7-874e-99260e77b9cb", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "fc23d4b8-76c5-4e59-9305-10846b8bcefe", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + }, + { + "id": "08f06ba5-3e60-4a0a-aaf9-f70bfc7ae99e", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e28227ee-cb54-4557-8908-01864f80055f", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "364a632f-b66a-4ca4-8bbe-ec2ce1af9df8", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "42cad815-4de0-4b67-abca-7f7aaf55e589", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "e3302def-d387-465c-a420-7ab01570e94a", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "63a3cb24-b124-428e-ac0f-253eb1fe485d", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2eb041ca-970a-45fd-a167-2a497579bc8c", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "2c02e9ce-7d86-4a5b-84b8-cf93114ddf26", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "2d39b55e-46c4-4dec-bd83-f081c708f544", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "e869fffd-f801-492d-a6c7-d6c6143817e5", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "daeb863b-4773-4668-98fb-403e93414eb2", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "a98b9f93-ec39-4f3c-acb7-cd92161e3717", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "c59a379a-3934-4e6f-be20-1803b0786d97", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0a299a91-277c-4f38-95e7-6c520f892b63", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "d51531b3-a8ea-44e2-a48f-69991f9166cc", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "b4b33a89-01db-468e-9a4e-c5ac58304fed", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "d1727e08-fb90-49ce-bb7e-d7a55a50ee64", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "564ae79d-e505-416c-b794-ddd3a3c21fde", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "986d2d9e-0d0d-4317-92b3-a7a8d9bec4de", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "e14ec2e9-0d24-4960-8779-00f769ccc01b", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "8f044609-b615-4522-b9e7-8361cb08b0b3", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "341b838d-ba26-4280-b0af-3e5d3403c938", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "26a17e7e-1a6e-439f-a54a-05a63d1c91fb", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "2489b2c0-5b3a-4404-8428-be4ce653da72", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "8c7e1d96-bf79-42e6-9360-b5e7b8dddc8d", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "01f959d8-123f-4263-ad6e-386e8b4d0e05", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "b47a30d6-3c49-4bbe-b15e-b0eb6cffc0f3", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "2cc8166d-6d77-4a85-9945-bc22b0f550e3", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "1d87a800-cc11-4d85-aa76-8a6d828e2269", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "42cc1538-f83a-4a94-b5a5-d16b80824a02", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "13ff6325-822e-4087-9e74-086de77fe89e", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e501527e-dec8-4fde-a539-8e77d86b5081", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "b4b519d8-070d-4dab-854e-d6e3b2b36205", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + } + ], + "defaultDefaultClientScopes": [ + "role_list", + "profile", + "email", + "roles", + "web-origins" + ], + "defaultOptionalClientScopes": [ + "offline_access", + "address", + "phone", + "microprofile-jwt" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "ed42958b-6e78-42a9-9f40-2e40bd6c8dd0", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-attribute-mapper", + "saml-user-property-mapper", + "oidc-usermodel-property-mapper", + "oidc-address-mapper", + "saml-role-list-mapper", + "oidc-full-name-mapper" + ] + } + }, + { + "id": "572219a7-3053-4940-87c5-ad94a6fb6dd3", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "8704a420-bf90-4e12-9e33-d21f39a2385b", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "8cf98455-916b-487a-8322-3f5d283400c2", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "0b43488d-108b-41f5-ab6d-56a4ac8ff63c", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-role-list-mapper", + "oidc-address-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "6f0ebf9b-900a-4ca9-8fea-90719f218689", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "b5a486b3-abf9-49a1-8dc6-dc5e20776681", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "be43420e-8ffc-4f53-b745-f2f0cd88f000", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "f749bd77-72f2-4dc4-a65e-dd89b255f12f", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "a541cbb1-8a27-4061-a389-9f24ba1c2eb1", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "18504bf7-63f1-4848-b565-6348fa6b0048", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "497d8386-9a74-4b7b-a4e6-78bbbbb5d795", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "d964973c-2106-4db3-a814-f7a34ae7a1ce", + "alias": "Authentication Options", + "description": "Authentication options.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ff1b250-85b1-4709-8719-3eabcb34493f", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "43f683be-52e7-43cd-aa9e-6318b8079ad0", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "5195e46e-b2c3-49e3-8987-db8b19c45fc5", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "aca5480d-842c-4fa9-aff1-b8af8d51d82a", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Account verification options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "66a00ea9-7ec1-4450-905a-14b7f3f8e4bf", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "a24445a4-1988-4b5b-bde6-fa36dbd07e03", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "31c2bc3b-6eb1-4c4f-8464-3528f7445ef7", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4ccc8da9-0e1e-4f30-99c0-e2139f671a80", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "flowAlias": "forms", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "fa1183d1-af7a-40dd-ba85-d7d37867639c", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "f5b5e49b-7cc9-4011-b9fa-60f0ef65e735", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "5cf727d2-e25a-4c88-a55b-4eea9134adb1", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ec4f009-2f16-464b-8feb-a0bdc0dad195", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "User creation or linking", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "a695a5e0-326f-4658-8518-a1769d97ad5f", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4611e0a9-a6a4-4e32-8500-e68877b464b1", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Authentication Options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "55be727b-a17b-40c5-a5c3-c2d72c7f54cb", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "flowAlias": "registration form", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "00ec3b72-3abb-4db3-ad2f-595bc2f7e086", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "82e4f3a3-744b-4d8a-8785-6eabaf9e05c9", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "0c0312b2-db7c-433c-ab15-20b18bfb5f4a", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "ee0faa63-999c-42e6-8189-c22a5cc14dc5", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "506eed8f-88c9-4978-b13a-886f1efc45c0", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "oauth2DevicePollingInterval": "5", + "clientOfflineSessionMaxLifespan": "0", + "clientSessionIdleTimeout": "0", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5" + }, + "keycloakVersion": "14.0.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} \ No newline at end of file diff --git a/charts/keycloak/templates/NOTES.txt b/charts/keycloak/templates/NOTES.txt new file mode 100644 index 0000000..f32bde6 --- /dev/null +++ b/charts/keycloak/templates/NOTES.txt @@ -0,0 +1,36 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "keycloak.fullname" . }} ({{ .Values.replicaCount }} replica(s)) + - Service/{{ include "keycloak.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "keycloak.envConfigMapName" . }} +{{- if include "keycloak.renderRealmConfigMap" . }} + - ConfigMap/{{ include "keycloak.realmConfigMapName" . }} (realm import) +{{- end }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "keycloak.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "keycloak.fullname" . }} +{{- end }} + +Database: {{ .Values.database.user }}@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }} + +Give the registry chart this URL: + keycloak.url: {{ include "keycloak.url" . }} + +{{ if .Values.realmImport.enabled }} +NEXT STEP - the registry cannot authenticate until you do this: + + The realm export ships with a masked admin-api client secret, so it is not a + working credential. Once Keycloak is up: + + 1. kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "keycloak.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + 2. Open http://localhost:{{ .Values.service.port }}/auth/admin and sign in as "{{ .Values.admin.username }}". + 3. Realm "sunbird-rc" -> Clients -> admin-api -> Credentials -> Regenerate Secret. + 4. Store that value, and set it as the registry's admin client secret. +{{- end }} + +Watch the rollout (first start imports the realm and migrates the schema, so +give it a few minutes): + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "keycloak.fullname" . }} diff --git a/charts/keycloak/templates/_helpers.tpl b/charts/keycloak/templates/_helpers.tpl new file mode 100644 index 0000000..8557b64 --- /dev/null +++ b/charts/keycloak/templates/_helpers.tpl @@ -0,0 +1,159 @@ +{{/* +# ============================================================================ +# KEYCLOAK (SUNBIRD RC) CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the realm-import +# and legacy-Keycloak database wiring this chart needs. +# ============================================================================ +*/}} + +{{- define "keycloak.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "keycloak.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "keycloak.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "keycloak.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "keycloak.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "keycloak.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "keycloak.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +The base URL other services use to reach this Keycloak, including the /auth +context path the legacy distribution serves under. The registry's issuer check +compares against exactly this, so it must match what the registry is given. +*/}} +{{- define "keycloak.url" -}} +{{- printf "http://%s.%s.svc.cluster.local:%v/auth" (include "keycloak.fullname" .) .Release.Namespace .Values.service.port -}} +{{- end }} + +{{/* +Name of the ConfigMap holding the realm export. + +The chart renders its own by default. An existingConfigMap wins, for the case +where the realm is managed outside the chart entirely. +*/}} +{{- define "keycloak.realmConfigMapName" -}} +{{- if .Values.realmImport.existingConfigMap -}} +{{- .Values.realmImport.existingConfigMap -}} +{{- else -}} +{{- printf "%s-realm" (include "keycloak.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Emits "true" when this chart renders the realm ConfigMap itself, i.e. whenever +the realm is not being supplied by an out-of-band ConfigMap. +*/}} +{{- define "keycloak.renderRealmConfigMap" -}} +{{- if and .Values.realmImport.enabled (not .Values.realmImport.existingConfigMap) -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +The realm JSON: an inline override if given, otherwise the file shipped in the +chart under files/. +*/}} +{{- define "keycloak.realmJson" -}} +{{- if .Values.realmImport.realmJson -}} +{{- .Values.realmImport.realmJson -}} +{{- else -}} +{{- $json := .Files.Get .Values.realmImport.file -}} +{{- if not $json -}} +{{- fail (printf "%s: realmImport is enabled but %q is not present in the chart, and neither realmImport.realmJson nor realmImport.existingConfigMap is set." .Chart.Name .Values.realmImport.file) -}} +{{- end -}} +{{- $json -}} +{{- end -}} +{{- end }} + +{{/* +Path the realm file is mounted at, which is what KEYCLOAK_IMPORT points to. +*/}} +{{- define "keycloak.realmImportPath" -}} +{{- printf "%s/%s" (.Values.realmImport.mountPath | trimSuffix "/") .Values.realmImport.fileName -}} +{{- end }} + +{{/* +Environment for the legacy (WildFly) Keycloak distribution. This image predates +Keycloak's KC_* variables, so it takes DB_VENDOR/DB_ADDR and friends. Values +mirror the compose stack. +*/}} +{{- define "keycloak.env" -}} +{{- $db := .Values.database }} +{{- if not $db.host }} +{{- fail (printf "%s: database.host is required - point it at the PostgreSQL primary service, e.g. registry-db-rw" .Chart.Name) }} +{{- end }} +{{- if not $db.passwordSecret.name }} +{{- fail (printf "%s: database.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) }} +{{- end }} +{{- if not .Values.admin.passwordSecret.name }} +{{- fail (printf "%s: admin.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) }} +{{- end }} +- name: DB_VENDOR + value: {{ $db.vendor | quote }} +- name: DB_ADDR + value: {{ $db.host | quote }} +- name: DB_PORT + value: {{ $db.port | quote }} +- name: DB_DATABASE + value: {{ $db.name | quote }} +- name: DB_USER + value: {{ $db.user | quote }} +{{- with $db.schema }} +- name: DB_SCHEMA + value: {{ . | quote }} +{{- end }} +- name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $db.passwordSecret.name }} + key: {{ $db.passwordSecret.key }} +- name: KEYCLOAK_USER + value: {{ .Values.admin.username | quote }} +- name: KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.admin.passwordSecret.name }} + key: {{ .Values.admin.passwordSecret.key }} +- name: PROXY_ADDRESS_FORWARDING + value: {{ .Values.proxyAddressForwarding | quote }} +{{- if .Values.realmImport.enabled }} +- name: KEYCLOAK_IMPORT + value: {{ include "keycloak.realmImportPath" . | quote }} +{{- end }} +{{- with (include "oan-common.env" . | trim) }} +{{ . }} +{{- end }} +{{- end }} + +{{/* +Dependency waits, derived from this chart's own settings so there is nothing to +keep in sync: the host comes from database.host, which is the same value the +container connects to. +*/}} +{{- define "keycloak.waitFor" -}} +{{- $tcp := list -}} +{{- if .Values.waitFor.database }} +{{- $tcp = append $tcp (dict "name" "database" "host" .Values.database.host "port" .Values.database.port) -}} +{{- end -}} +{{- $tcp = concat $tcp (.Values.waitFor.extraTcp | default list) -}} +{{- include "oan-common.waitFor" (dict "ctx" . "tcp" $tcp "http" (.Values.waitFor.extraHttp | default list)) -}} +{{- end }} diff --git a/charts/keycloak/templates/configmap-realm.yaml b/charts/keycloak/templates/configmap-realm.yaml new file mode 100644 index 0000000..6890099 --- /dev/null +++ b/charts/keycloak/templates/configmap-realm.yaml @@ -0,0 +1,27 @@ +{{- if include "keycloak.renderRealmConfigMap" . }} +{{/* +The realm the registry expects: realm `sunbird-rc`, the admin-api and +registry-frontend clients, and the roles the registry checks. Mounted at +realmImport.mountPath and imported on startup via KEYCLOAK_IMPORT. + +The exported admin-api client secret is masked ("**********"), not a working +credential - it has to be regenerated in the Keycloak console after the first +import. See this chart's README. + +Not rendered when realmImport.existingConfigMap points at a ConfigMap managed +outside the chart. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "keycloak.realmConfigMapName" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{ .Values.realmImport.fileName }}: |- + {{- include "keycloak.realmJson" . | nindent 4 }} +{{- end }} diff --git a/charts/keycloak/templates/configmap.yaml b/charts/keycloak/templates/configmap.yaml new file mode 100644 index 0000000..ed67084 --- /dev/null +++ b/charts/keycloak/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "keycloak.envConfigMapName" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with (include "oan-common.envConfigMapData" . | trim) }} +data: + {{- . | nindent 2 }} +{{- else }} +data: {} +{{- end }} diff --git a/charts/keycloak/templates/deployment.yaml b/charts/keycloak/templates/deployment.yaml new file mode 100644 index 0000000..bca671f --- /dev/null +++ b/charts/keycloak/templates/deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + # The legacy distribution is not clustered here, and the realm import runs on + # startup, so replicas are rolled one at a time with the old pod gone first. + strategy: + type: Recreate + selector: + matchLabels: + {{- include "keycloak.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "keycloak.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + {{- if include "keycloak.renderRealmConfigMap" . }} + {{/* + Rolls the pod when the realm changes. Only possible while the chart owns + the realm content - with realmImport.existingConfigMap, Helm cannot read + cluster state to checksum it, so restart Keycloak yourself after editing + that ConfigMap. + */}} + checksum/realm: {{ include "keycloak.realmJson" . | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "keycloak.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + {{- $init := include "keycloak.waitFor" . | trim }} + {{- $extraInit := "" }} + {{- with .Values.extraInitContainers }} + {{- $extraInit = toYaml . | trim }} + {{- end }} + {{- if or $init $extraInit }} + initContainers: + {{- with $init }} + {{- . | nindent 8 }} + {{- end }} + {{- with $extraInit }} + {{- . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "keycloak.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- include "keycloak.env" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "keycloak.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + {{- if or .Values.realmImport.enabled .Values.extraVolumeMounts }} + volumeMounts: + {{- if .Values.realmImport.enabled }} + - name: realm + mountPath: {{ .Values.realmImport.mountPath }} + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.realmImport.enabled .Values.extraVolumes }} + volumes: + {{- if .Values.realmImport.enabled }} + - name: realm + configMap: + name: {{ include "keycloak.realmConfigMapName" . }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/keycloak/templates/ingress.yaml b/charts/keycloak/templates/ingress.yaml new file mode 100644 index 0000000..1e99eb8 --- /dev/null +++ b/charts/keycloak/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "keycloak.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/keycloak/templates/poddisruptionbudget.yaml b/charts/keycloak/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..f210bd2 --- /dev/null +++ b/charts/keycloak/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Caps how many pods can be voluntarily disrupted at once - node drains, cluster +upgrades, autoscaler scale-down. + +With replicaCount: 1 a PDB of minAvailable: 1 blocks drains entirely, because +evicting the only pod would breach it. That is why this is disabled by default +and should be enabled together with replicaCount >= 2. +*/}} +{{- if and (eq (int .Values.replicaCount) 1) (not .Values.podDisruptionBudget.allowSingleReplica) }} +{{- fail (printf "%s: podDisruptionBudget is enabled with replicaCount 1, which blocks node drains entirely. Raise replicaCount to 2+, or set podDisruptionBudget.allowSingleReplica=true if you accept that." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "keycloak.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/keycloak/templates/service.yaml b/charts/keycloak/templates/service.yaml new file mode 100644 index 0000000..1a824dd --- /dev/null +++ b/charts/keycloak/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "keycloak.selectorLabels" . | nindent 4 }} diff --git a/charts/keycloak/templates/serviceaccount.yaml b/charts/keycloak/templates/serviceaccount.yaml new file mode 100644 index 0000000..83b8800 --- /dev/null +++ b/charts/keycloak/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "keycloak.serviceAccountName" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/keycloak/templates/tests/test-connection.yaml b/charts/keycloak/templates/tests/test-connection.yaml new file mode 100644 index 0000000..5c31958 --- /dev/null +++ b/charts/keycloak/templates/tests/test-connection.yaml @@ -0,0 +1,39 @@ +{{- if .Values.tests.enabled }} +{{/* +`helm test` check: Keycloak is serving its realm endpoint. + +This is the check that matters for the stack - the registry validates tokens +against this exact URL, so if it does not answer, the registry cannot +authenticate no matter how healthy the pod looks. +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "keycloak.fullname" . }}-test-connection + labels: + {{- include "keycloak.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: test + image: {{ printf "%s/%s:%s" .Values.tests.image.registry .Values.tests.image.repository .Values.tests.image.tag | quote }} + command: + - /bin/sh + - -c + - | + set -e + base="http://{{ include "keycloak.fullname" . }}:{{ .Values.service.port }}/auth" + echo "GET $base/" + wget -q -O /dev/null --timeout=10 "$base/" + echo "OK: /auth is serving" + {{- if .Values.realmImport.enabled }} + echo "GET $base/realms/{{ .Values.tests.realm }}" + wget -q -O /dev/null --timeout=10 "$base/realms/{{ .Values.tests.realm }}" + echo "OK: realm {{ .Values.tests.realm }} is imported and serving" + {{- end }} + resources: + {{- toYaml .Values.tests.resources | nindent 8 }} +{{- end }} diff --git a/charts/keycloak/values.yaml b/charts/keycloak/values.yaml new file mode 100644 index 0000000..0391081 --- /dev/null +++ b/charts/keycloak/values.yaml @@ -0,0 +1,313 @@ +# ============================================================================ +# keycloak - default values +# +# Keycloak for the OAN registry, on the Sunbird RC Keycloak image. This is the +# legacy (WildFly/JBoss) Keycloak distribution: it serves under /auth and takes +# DB_VENDOR-style environment variables, not the modern KC_* ones. +# +# Defaults mirror the verified compose stack (registry/docker-compose.yml) so a +# cluster deploy reproduces what was tested locally. Where this chart deviates, +# it says so and why. +# +# Minimum you must set: +# - database.host (e.g. the CNPG primary, registry-db-rw) +# - database.passwordSecret.name +# - admin.passwordSecret.name +# ============================================================================ + +replicaCount: 1 + +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-keycloak + # Compose uses `latest`. A chart must not deploy a moving tag, so this pins + # the version Sunbird's own v2 deployment pairs with sunbird-rc-core v2.0.0. + tag: "v1.0.0" + # Digest pin; wins over tag. Preferred for anything past dev. + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +# Only needed when the release name differs from "keycloak". The resulting name +# is the in-cluster DNS the registry connects to, and the registry validates the +# token issuer against exactly this host - so keep it stable once anything +# depends on it. +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: true + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + annotations: {} +# Compose also publishes 9990, the WildFly management port. That exists only for +# the compose healthcheck; nothing in-cluster needs it, and the Keycloak admin +# console is on 8080 under /auth/admin. Not exposed here. + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: keycloak.local + paths: + # Legacy Keycloak serves everything under /auth + - path: /auth + pathType: Prefix + tls: [] + +# --------------------------------------------------------------------------- +# Database +# +# Defaults match compose: Keycloak shares the registry's database and connects +# as the same user. Sunbird's own Helm charts instead give Keycloak its own +# database ("keycloak") on the same host, which is the better arrangement - +# Keycloak's realm tables have no reason to sit beside registry records, and it +# frees the registry to connect as a non-superuser owner. To separate them: +# +# database: +# name: keycloak +# user: keycloak +# +# and provision that database and role in the cluster. +# --------------------------------------------------------------------------- +database: + vendor: postgres + # PostgreSQL primary service, e.g. "registry-db-rw" + host: "" + port: 5432 + name: registry + user: postgres + schema: "" + # Secret holding the database password. Required. + passwordSecret: + name: "" + key: password + +# --------------------------------------------------------------------------- +# Keycloak admin user (the console login) +# --------------------------------------------------------------------------- +admin: + username: admin + passwordSecret: + name: "" + key: keycloakAdminPassword + +# --------------------------------------------------------------------------- +# Realm import +# +# The chart ships the realm the registry expects: realm `sunbird-rc`, the +# admin-api and registry-frontend clients, and the roles the registry checks. +# The legacy image imports it on startup when the realm is not already present. +# +# IMPORTANT: the exported admin-api client secret is masked ("**********"), not +# a working credential. After the first import, regenerate it in the console +# (realm -> Clients -> admin-api -> Credentials -> Regenerate Secret) and hand +# that value to the registry chart. First install is therefore two-phase. +# --------------------------------------------------------------------------- +realmImport: + enabled: true + + # The realm shipped with this chart, rendered into a ConfigMap and mounted for + # KEYCLOAK_IMPORT. Copied verbatim from the compose stack's + # registry/imports/realm-export.json, so the cluster imports the same realm + # that was verified locally. + # + # Keeping it in the chart means `helm install` needs nothing prepared + # beforehand, and a realm change rolls the pod via the checksum annotation. + # The trade-off is that this is a second copy of that file: if the compose one + # changes, update this one too. + file: files/realm-export.json + + # Key in the ConfigMap, which becomes the filename on disk and the value of + # KEYCLOAK_IMPORT. + fileName: realm-export.json + + # Where the legacy image looks for imports. + mountPath: /opt/jboss/keycloak/imports + + # Use a ConfigMap managed outside this chart instead. When set, the chart + # renders no realm ConfigMap and mounts this name - so it must already exist. + existingConfigMap: "" + + # Inline realm JSON, overriding the shipped file. Useful for a test with a + # deliberately minimal realm. + realmJson: "" + +# Lets a caller behind an ingress obtain a token whose `iss` matches what the +# registry validates, via X-Forwarded-Host / X-Forwarded-Proto. Matches compose. +# If you enable ingress, the controller must actually set those headers. +proxyAddressForwarding: true + +# WildFly-based Keycloak is memory-hungry; Sunbird requests 500m/2G with no +# ceiling. Limits are mandatory in this repo, so it gets one. +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +# --------------------------------------------------------------------------- +# Probes +# +# These follow Sunbird's own Helm chart rather than the compose healthcheck: +# compose curls the WildFly management port (9990), which this chart does not +# expose. Readiness instead checks that Keycloak is really serving /auth. +# +# First start imports the realm and runs schema migrations, so startupProbe +# carries the slow path and liveness stays tight. +# --------------------------------------------------------------------------- +startupProbe: + enabled: true + httpGet: + path: /auth/ + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 30 + +livenessProbe: + enabled: true + tcpSocket: + port: http + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 10 + +readinessProbe: + enabled: true + httpGet: + path: /auth/ + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 10 + +# This image runs as the jboss user and writes into its own directories, so +# readOnlyRootFilesystem needs extra volumes to work. Off by default. +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# --------------------------------------------------------------------------- +# Dependency waiting +# +# Kubernetes has no equivalent of compose's `depends_on: condition: +# service_healthy`. Without these init containers the pod starts before its +# dependencies are up, fails, and crashloops with backoff - which recovers on +# its own but makes a first install look broken. +# --------------------------------------------------------------------------- +waitFor: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + timeoutSeconds: 300 + intervalSeconds: 3 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Wait for the database. Host and port come from `database` above, so there is + # nothing to keep in sync. + database: true + # Extra checks, if this Keycloak depends on anything else. + extraTcp: [] + extraHttp: [] + +# Extra init containers, appended after the waitFor ones. Rendered verbatim. +extraInitContainers: [] + +# Extra volumes and mounts, rendered verbatim. For certificates, extra config, +# or a writable scratch directory when readOnlyRootFilesystem is enabled. +extraVolumes: [] +extraVolumeMounts: [] + +# --------------------------------------------------------------------------- +# PodDisruptionBudget +# +# Caps voluntary disruptions - node drains, cluster upgrades, autoscaler +# scale-down. Disabled by default because with a single replica a PDB of +# minAvailable: 1 blocks drains entirely; the render fails if you enable it +# anyway without acknowledging that. +# --------------------------------------------------------------------------- +podDisruptionBudget: + enabled: false + minAvailable: 1 + # Set one or the other, not both. + maxUnavailable: "" + # Enable a PDB with a single replica anyway, accepting that node drains block. + allowSingleReplica: false + +# --------------------------------------------------------------------------- +# `helm test` checks +# +# helm test -n +# +# Runs after install to confirm the service actually answers, which is the +# difference between "the pod is running" and "the deployment works". +# --------------------------------------------------------------------------- +tests: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Realm the test expects to find. Only used when realmImport.enabled. + realm: sunbird-rc + + +# Extra non-secret configuration, exposed as a ConfigMap and injected with +# envFrom. The database and admin settings above render as explicit env vars, +# so they do not belong here. +envConfig: {} + # JAVA_OPTS_APPEND: "-Xms512m -Xmx1024m" + +# Extra env vars from specific Secret keys, beyond the two wired above. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} From 02c3c65238f90c28145bd4bb00a96da550348533 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:35:20 +0530 Subject: [PATCH 16/81] feat: add registry and discovery service charts [#53] registry runs Sunbird RC with its Participant schema mounted from a ConfigMap, and connects as the owner of the registry database rather than as postgres. discovery renders the Beckn discovery service, taking its whole DSN from the CNPG-generated Secret so no password is assembled or escaped in the chart. It enables readOnlyRootFilesystem, which the other service charts cannot yet. The Beckn spec is fetched by URL with a cache fallback, and the render fails when neither a URL nor an existing ConfigMap is set, since the service refuses to boot without the document. Both charts fail the render on an unset credential rather than defaulting one. --- charts/discovery/.helmignore | 13 + charts/discovery/CHANGELOG.md | 82 ++++ charts/discovery/Chart.yaml | 29 ++ charts/discovery/README.md | 320 ++++++++++++++ charts/discovery/ci/lint-values.yaml | 20 + charts/discovery/ci/url-secret-values.yaml | 53 +++ charts/discovery/examples/discovery.dev.yaml | 85 ++++ charts/discovery/examples/discovery.prod.yaml | 122 +++++ charts/discovery/templates/NOTES.txt | 59 +++ charts/discovery/templates/_helpers.tpl | 178 ++++++++ charts/discovery/templates/configmap.yaml | 16 + charts/discovery/templates/deployment.yaml | 111 +++++ charts/discovery/templates/hpa.yaml | 54 +++ charts/discovery/templates/ingress.yaml | 41 ++ .../templates/poddisruptionbudget.yaml | 33 ++ charts/discovery/templates/service.yaml | 19 + .../discovery/templates/serviceaccount.yaml | 13 + .../templates/tests/test-connection.yaml | 40 ++ charts/discovery/values.yaml | 416 ++++++++++++++++++ charts/registry/.helmignore | 13 + charts/registry/CHANGELOG.md | 74 ++++ charts/registry/Chart.yaml | 26 ++ charts/registry/README.md | 244 ++++++++++ charts/registry/ci/lint-values.yaml | 18 + charts/registry/examples/registry.dev.yaml | 84 ++++ charts/registry/examples/registry.prod.yaml | 144 ++++++ .../registry/files/schemas/Participant.json | 170 +++++++ charts/registry/templates/NOTES.txt | 35 ++ charts/registry/templates/_helpers.tpl | 181 ++++++++ .../registry/templates/configmap-schemas.yaml | 18 + charts/registry/templates/configmap.yaml | 16 + charts/registry/templates/deployment.yaml | 108 +++++ charts/registry/templates/hpa.yaml | 51 +++ charts/registry/templates/ingress.yaml | 41 ++ .../templates/poddisruptionbudget.yaml | 33 ++ charts/registry/templates/service.yaml | 19 + charts/registry/templates/serviceaccount.yaml | 13 + .../templates/tests/test-connection.yaml | 36 ++ charts/registry/values.yaml | 353 +++++++++++++++ 39 files changed, 3381 insertions(+) create mode 100644 charts/discovery/.helmignore create mode 100644 charts/discovery/CHANGELOG.md create mode 100644 charts/discovery/Chart.yaml create mode 100644 charts/discovery/README.md create mode 100644 charts/discovery/ci/lint-values.yaml create mode 100644 charts/discovery/ci/url-secret-values.yaml create mode 100644 charts/discovery/examples/discovery.dev.yaml create mode 100644 charts/discovery/examples/discovery.prod.yaml create mode 100644 charts/discovery/templates/NOTES.txt create mode 100644 charts/discovery/templates/_helpers.tpl create mode 100644 charts/discovery/templates/configmap.yaml create mode 100644 charts/discovery/templates/deployment.yaml create mode 100644 charts/discovery/templates/hpa.yaml create mode 100644 charts/discovery/templates/ingress.yaml create mode 100644 charts/discovery/templates/poddisruptionbudget.yaml create mode 100644 charts/discovery/templates/service.yaml create mode 100644 charts/discovery/templates/serviceaccount.yaml create mode 100644 charts/discovery/templates/tests/test-connection.yaml create mode 100644 charts/discovery/values.yaml create mode 100644 charts/registry/.helmignore create mode 100644 charts/registry/CHANGELOG.md create mode 100644 charts/registry/Chart.yaml create mode 100644 charts/registry/README.md create mode 100644 charts/registry/ci/lint-values.yaml create mode 100644 charts/registry/examples/registry.dev.yaml create mode 100644 charts/registry/examples/registry.prod.yaml create mode 100644 charts/registry/files/schemas/Participant.json create mode 100644 charts/registry/templates/NOTES.txt create mode 100644 charts/registry/templates/_helpers.tpl create mode 100644 charts/registry/templates/configmap-schemas.yaml create mode 100644 charts/registry/templates/configmap.yaml create mode 100644 charts/registry/templates/deployment.yaml create mode 100644 charts/registry/templates/hpa.yaml create mode 100644 charts/registry/templates/ingress.yaml create mode 100644 charts/registry/templates/poddisruptionbudget.yaml create mode 100644 charts/registry/templates/service.yaml create mode 100644 charts/registry/templates/serviceaccount.yaml create mode 100644 charts/registry/templates/tests/test-connection.yaml create mode 100644 charts/registry/values.yaml diff --git a/charts/discovery/.helmignore b/charts/discovery/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/discovery/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/discovery/CHANGELOG.md b/charts/discovery/CHANGELOG.md new file mode 100644 index 0000000..873dda1 --- /dev/null +++ b/charts/discovery/CHANGELOG.md @@ -0,0 +1,82 @@ +# Changelog + +All notable changes to the `discovery` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-09-01 + +Initial release. Configuration ported from the verified +`discovery-service/docker-compose.yml` stack, built on the `oan-common` library. + +### Added +- Deployment, Service, env ConfigMap, ServiceAccount, and optional Ingress, + HorizontalPodAutoscaler and PodDisruptionBudget. +- Two ways to supply `DATABASE_URL`, exactly one of which must be set: + `database.urlSecret` for the whole DSN from one Secret key — which is what + CloudNativePG's generated `-app` Secret already provides under `uri` + — or `database.host/port/name/user/sslMode` plus `passwordSecret`, from which + the chart assembles a DSN with the password injected through Kubernetes' + `$(VAR)` expansion so it stays in the Secret and out of the rendered manifest. +- Every other environment variable derived from structured values rather than + restated, so `SERVER_PORT` cannot drift from the container port and + `DATABASE_URL` cannot drift from `database.*`. +- Beckn specification wiring: `becknSpec.url` for the fetch path, + `becknSpec.existingConfigMap` for the air-gapped one, and a volume mounted at + the cache path either way — `/app` is not writable by uid 65532, so with + `readOnlyRootFilesystem` on the fetch would have nowhere to cache. +- Split probes: liveness and startup on `/healthz`, readiness on `/readyz`. + Liveness on `/readyz` would restart every pod during a database blip, turning + a recoverable outage into a crashloop across the Deployment. +- Render-time validation of `app.networkId`, the database DSN, the Beckn spec + source, and the endpoint that a non-default `embeddings.provider` or + `otel.exporter` requires. Each names the value and why the boot needs it. +- `helm test` check against `/readyz` rather than `/healthz`: `/healthz` is 200 + with the database down and would pass on a deployment that cannot serve a + single discover. +- `extraVolumes` / `extraVolumeMounts` / `extraInitContainers` escape hatches. +- No init container waiting on PostgreSQL, unlike the `registry` chart. The + service fails fast when it cannot connect and Kubernetes' restart backoff is + already the retry, so a wait container buys a tidier first install at the cost + of a second thing to configure and keep pointed at the right host. +- Per-environment example values for dev and production. Production mounts the + Beckn document from a ConfigMap rather than fetching it, sets a + PodDisruptionBudget and pod anti-affinity, and leaves `image.repository` as a + documented TODO. + +### Changed from the compose stack +- `DATABASE_AUTO_MIGRATE` defaults to `false`. The compose file states the + intent directly: migrating is a step someone decides to take. The README + documents the scale-to-one-replica upgrade that does it. +- The rate limiter is left unset, so the service's own 20 rps / 40 burst + defaults apply. Compose's 100000/100000 is a local-stack ceiling, not a + deployment value. +- `envConfig` is empty. The image already carries the reviewed + `config/common.yaml` defaults and the environment layer sits above it; + restating them here would be a second copy of decisions this chart did not + make. +- Both security contexts are on by default, unlike the other OAN service + charts. The image is distroless/static as uid 65532, so + `readOnlyRootFilesystem` costs nothing once the spec cache is a volume. +- `automountServiceAccountToken` is false: the service calls no Kubernetes API. + +### Known prerequisite +- PostgreSQL must have **pgvector**, and must create the `vector` extension at + bootstrap. `postgresql-cnpg` 0.2.1 ships + `examples/discovery-db.dev.yaml`, which does both: it pins + `dhi.io/pgvector:0.8-pg16` and creates the extension in + `postInitApplicationSQL`. + + The bootstrap half is not optional. `vector` is not a trusted extension, so + the database owner this service connects as cannot create it - only a + superuser can, and `enableSuperuserAccess` is false. Without it the first + statement of the migration fails and the pod crashloops on an error that + reads like a credentials problem. `pg_trgm` is trusted and would have + succeeded, which is what makes the failure look selective and confusing. + + Neither condition is detectable at render time; the README describes both + failure modes. +- No image is published for `discovery-service` — its CI builds and scans one + but pushes nothing — so `image.repository` is empty and the render fails until + an environment supplies a tag. diff --git a/charts/discovery/Chart.yaml b/charts/discovery/Chart.yaml new file mode 100644 index 0000000..8bed016 --- /dev/null +++ b/charts/discovery/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: discovery +description: >- + The OAN Beckn v2.0.0 discover-and-publish service. Holds published catalogs + and answers discover queries over them. PostgreSQL is its only datastore, and + it needs the `vector` and `pg_trgm` extensions. +type: application +version: 0.1.0 +# No image is published for this service yet - CI builds and scans one but +# pushes nothing - so this is the version the chart deploys by default once one +# exists, and `image.repository` is empty until then. See values.yaml. +appVersion: "0.1.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - discovery + - publish + - beckn +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/OpenAgriNet/discovery-service +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/discovery/README.md b/charts/discovery/README.md new file mode 100644 index 0000000..4fa3efa --- /dev/null +++ b/charts/discovery/README.md @@ -0,0 +1,320 @@ +# discovery + +The OAN [discover-and-publish +service](https://github.com/OpenAgriNet/discovery-service) — a Beckn v2.0.0 +`/publish` and `/discover` implementation in Go. PostgreSQL is its only +datastore; there is no spatial extension and no separate search engine. + +Configuration is ported from the verified `discovery-service/docker-compose.yml` +stack, with the two settings that file marks as local-only put back to their +deployment values (see [Differences from compose](#differences-from-compose)). + +## What it renders + +| Resource | Notes | +|---|---| +| Deployment | Probes on `/healthz` and `/readyz`, resources mandatory | +| Service | `ClusterIP` on 8080 | +| ConfigMap (env) | Overrides from `envConfig`. Empty by default — see [Configuration](#configuration) | +| ServiceAccount | `automountServiceAccountToken: false` — the service calls no Kubernetes API | +| HorizontalPodAutoscaler | Optional, off by default | +| PodDisruptionBudget | Optional, off by default | +| Ingress | Optional, off by default | +| Test Pod | `helm test` check against `/readyz` | + +No Secret and no ConfigMap holding the Beckn document. Both are referenced by +name and created outside the chart. + +## Install + +```bash +# 1. The database — pgvector, plus `vector` created at bootstrap. Both matter; see below. +helm install discovery-db charts/postgresql-cnpg -n oan-discovery \ + -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml + +# 2. The service +helm install discovery charts/discovery -n oan-discovery \ + -f charts/discovery/examples/discovery.dev.yaml + +helm test discovery -n oan-discovery +``` + +## The database + +The first migration creates two extensions: + +```sql +CREATE EXTENSION IF NOT EXISTS vector; -- pgvector: the embedding column and its HNSW index +CREATE EXTENSION IF NOT EXISTS pg_trgm; -- contrib: trigram matching for lexical retrieval +``` + +No stock CloudNativePG operand image carries pgvector, so the cluster needs an +image that does. [`postgresql-cnpg`](../postgresql-cnpg) ships a ready example: + +```bash +helm install discovery-db charts/postgresql-cnpg -n oan-discovery \ + -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml +``` + +It pins `dhi.io/pgvector:0.8-pg16` — pgvector 0.8.6 and pg_trgm 1.6 on +PostgreSQL 16, amd64 and arm64. PostgreSQL **16** because that is what this +service is verified against: the design doc names it, `tests/dbtest` pins +`pgvector/pgvector:0.8.0-pg16`, and `EXPLAIN (GENERIC_PLAN)` in that suite is +PG16-and-later only. The schema does apply cleanly to 14, so this is about +staying on the tested major rather than a hard incompatibility. + +Two settings in that example are load-bearing: + +### `vector` is not a trusted extension + +`pg_trgm` is marked `trusted = true`, so a database owner may create it. +`vector` is not: + +``` +ERROR: permission denied to create extension "vector" +HINT: Must be superuser to create this extension. +``` + +This service connects as the bootstrap **owner**, not the superuser, and CNPG's +`enableSuperuserAccess` is false. So without help, the very first statement of +the migration fails — and the pod crashloops on an error that reads like a bad +password rather than a missing grant. + +The example creates both extensions in `bootstrap.postInitApplicationSQL`, which +CNPG runs as superuser inside the freshly created database. That is the one +moment superuser is available without granting it to anything long-lived; after +it, the service's own `IF NOT EXISTS` is a no-op and the rest of the migration +runs as the owner. + +### The image's postgres user is uid 70 + +CNPG defaults to 26 and applies it as the pod's `runAsUser` and `fsGroup`, so +the example sets `postgresUID`/`postgresGID` to 70. Left unset, the data +directory is handed to a user that does not exist in the image. + +### Not an operand image + +`dhi.io/pgvector` is a general-purpose PostgreSQL image. It satisfies CNPG's +documented requirements — `initdb`, `postgres`, `pg_ctl`, `pg_controldata`, +`pg_basebackup` and `du` are all on PATH, and CNPG overrides the image's own +entrypoint — but it is not built from `cloudnative-pg/postgres-containers` and +has not been run under the operator here. **Smoke-test the first cluster to +healthy before pointing this chart at it.** + +It also ships no barman-cloud, which rules out CNPG's in-core backup method. +The Barman Cloud Plugin brings its own sidecar and is unaffected. + +CNPG's generated app Secret carries `username`, `password`, `dbname`, `host`, +`port`, `uri` and `jdbc-uri`. `uri` is the whole DSN, which is why +`database.urlSecret` is the recommended way to wire this chart up. + +### Supplying the DSN + +Exactly one of these, or the render fails: + +```yaml +# Preferred — one Secret key holds the whole DSN. +database: + urlSecret: + name: discovery-db-app + key: uri +``` + +The DSN carries host, port, database and user, so `database.host` and the rest +go unread in this form. Leave them empty rather than setting a second copy that +nothing consumes and nothing keeps true. + +```yaml +# Assembled by the chart, with the password injected through Kubernetes' +# own $(VAR) expansion so it stays in the Secret. +database: + host: discovery-db-rw + port: 5432 + name: discovery + user: discovery + sslMode: prefer + passwordSecret: + name: discovery-db-app + key: password +``` + +The assembled form renders +`postgres://discovery:$(DATABASE_PASSWORD)@discovery-db-rw:5432/discovery?sslmode=prefer`, +and the kubelet substitutes the value of the `DATABASE_PASSWORD` env var — which +comes from the Secret — at container start. The password therefore never appears +in the rendered manifest, in `helm get manifest`, or in anything that logs a +template. + +The one caveat, and the reason `urlSecret` is preferred: that substitution is +textual, so a password containing `@ : / ? # %` must already be percent-encoded +in the Secret or it will silently produce a DSN that parses to the wrong thing. + +### Migrations + +The migrations are compiled into the binary. There is no Job, no sidecar and no +mounted directory: the service applies them on boot when `DATABASE_AUTO_MIGRATE` +is set, and treats "already at the latest version" as success. + +`database.autoMigrate` is **false** by default — migrating is a step someone +decides to take, not something that happens because a pod was rescheduled. + +```bash +helm upgrade discovery -n oan-discovery --reuse-values \ + --set database.autoMigrate=true --set replicaCount=1 +# verify the rollout, then upgrade back with autoMigrate=false +``` + +Scale to one replica first. The migration runs in-process on boot, so several +pods starting at once race to apply the same one. + +## The Beckn specification + +The service loads the Beckn v2.0.0 document **before** it serves and refuses to +start without it. The check is unconditional: turning L1 validation off does not +remove the requirement. + +It tries `becknSpec.url` first and falls back to the on-disk cache, so at least +one of `url` and `existingConfigMap` must be set. The render fails when both are +empty, because that configuration cannot boot. + +| Setting | What happens | +|---|---| +| `url` only | Fetched at boot and cached into an emptyDir. Needs egress, and refetches on every reschedule. The upstream URL tracks `main`, so this pins nothing. | +| `existingConfigMap` only | Mounted read-only at the cache path. No egress needed; reproducible. The boot logs one warning about the fetch it could not do — that warning is accurate and is not a failure. | +| both | Fetch first, ConfigMap as the fallback. | + +```bash +kubectl -n oan-discovery create configmap discovery-beckn-spec \ + --from-file=beckn.yaml=/tests/testdata/beckn-v2.0.0.yaml +``` + +The document is deliberately **not shipped in this chart**, for the same reason +the Dockerfile does not bake it into the image: a copy here is a second source of +truth that ages independently of the protocol. + +A volume is mounted at `becknSpec.cacheDir` either way — not only for the +air-gapped case. `/app` is not writable by uid 65532, and with +`readOnlyRootFilesystem` on, the fetch path would have nowhere to write its +cache. + +## Probes + +`/healthz` and `/readyz` answer different questions, and the split is +load-bearing: + +| Probe | Path | Behaviour | +|---|---|---| +| startup | `/healthz` | 60 × 5s. Covers the spec fetch, and migrations when they are on | +| liveness | `/healthz` | Depends on nothing. Stays 200 with the database down | +| readiness | `/readyz` | Pings PostgreSQL. 503 when it cannot reach it | + +Pointing liveness at `/readyz` would restart every pod during a database blip, +turning a recoverable outage into a crashloop across the whole Deployment. That +is why the two paths are different and why neither is configurable to the same +value by accident. + +There is no `exec` probe and no compose-style healthcheck: the runtime stage is +`gcr.io/distroless/static-debian12:nonroot`, which has no shell and no curl for +one to run. + +## Security context + +Both security contexts are **on by default**, unlike the other OAN service +charts. The image is distroless/static running as uid 65532 with a fully static +binary: there is no shell to escalate into, and the only path the process writes +is the spec cache, which is a mounted volume. `readOnlyRootFilesystem: true` +therefore costs nothing. + +If you add something that needs scratch space, add an `emptyDir` through +`extraVolumes` rather than turning this off. + +## Configuration + +`envConfig` is **empty by default**, and that is the deliberate part. + +The image already carries `config/common.yaml` — the project's reviewed defaults +for search sizing, validation, auth and the rest — and the environment layer sits +above it. Copying those values into this chart would create a second copy of +decisions the chart did not make, and it is the second copy that rots. + +Everything the chart genuinely owns is derived from structured values instead, so +`SERVER_PORT` cannot drift from the container port and `DATABASE_URL` cannot +drift from `database.*`: + +| Variable | Comes from | +|---|---| +| `DATABASE_URL` | `database.urlSecret`, or assembled from `database.host/port/name/user/sslMode` + `passwordSecret` | +| `DATABASE_AUTO_MIGRATE` | `database.autoMigrate` | +| `DATABASE_MAX_CONNS`, `DATABASE_MIN_CONNS` | `database.maxConns`, `database.minConns` | +| `APP_NETWORK_ID` | `app.networkId` — **required** | +| `APP_DEFAULT_TIMEZONE` | `app.defaultTimezone` | +| `SERVER_PORT` | `service.targetPort` | +| `LOG_LEVEL` | `logLevel` | +| `VALIDATION_SPEC_URL`, `VALIDATION_SPEC_CACHE_PATH` | `becknSpec.*` | +| `EMBEDDING_PROVIDER/MODEL/DIMENSIONS/ENDPOINT` | `embeddings.*` | +| `OTEL_EXPORTER`, `OTEL_EXPORTER_OTLP_ENDPOINT` | `otel.*` | +| `REPLICATION_TARGETS` | `replicationTargets`, joined with commas | + +Use `envConfig` to override a `common.yaml` default for one deployment. +Everything else the service reads: + +| Variable | Default | Notes | +|---|---|---| +| `SEARCH_DEFAULT_PAGE_SIZE` | 20 | What a request naming no limit gets | +| `SEARCH_MAX_PAGE_SIZE` | 100 | A larger limit is clamped to this | +| `SEARCH_MAX_CANDIDATES_PER_MODE` | 500 | Also the reachable pagination depth | +| `SEARCH_MAX_RADIUS_METERS` | 200000 | | +| `SEARCH_READ_DEADLINE` | 2s | | +| `SEARCH_FAIL_ON_UNAVAILABLE_MODE` | false | true turns a degraded mode into a 400 | +| `RATE_LIMIT_RPS` / `RATE_LIMIT_BURST` | 20 / 40 | `burst >= rps > 0`; a bucket smaller than one second's refill can never fill | +| `SERVER_SHUTDOWN_TIMEOUT` | 15s | | +| `SERVER_MAX_REQUEST_BODY_BYTES` | 10485760 | | +| `VALIDATION_ENABLE_L1_SCHEMA` | true | | +| `VALIDATION_ENABLE_L2_CONTEXT` | false | Nothing reads it; `true` refuses the boot | +| `AUTH_ENABLE_SIGNATURE_VERIFICATION` | false | Phase 2; `true` refuses the boot | +| `EXT_ALLOW_NETWORK_FETCH` | false | A configured URL is trusted, one from a request body is not | +| `GEO_RESOLUTION_CELLS` | 8 | H3 resolution | +| `ERROR_INCLUDE_LEGACY_TYPE` | false | | + +A key that matches no config field **fails the boot**, so a typo in `envConfig` +stops the service rather than being silently ignored. + +`envConfig` lands in a ConfigMap. Never put a secret there — use `secretEnv` or +`envFromSecrets`. + +## Differences from compose + +| | compose | chart | Why | +|---|---|---|---| +| `DATABASE_AUTO_MIGRATE` | `true` | `false` | Compose says it: "left false in deployment, where migrating is a step someone decides to take" | +| `RATE_LIMIT_RPS` / `BURST` | `100000` | unset (20 / 40) | Compose says it: "effectively off for the local stack… deployments leave both unset and get the real defaults back" | +| Beckn document | bind-mounted from the working tree | fetched, or from a ConfigMap | There is no working tree to mount from | +| `LOG_LEVEL` | `debug` | `info` | `debug` in the dev example | +| PostgreSQL | `pgvector/pgvector:0.8.0-pg16` | `dhi.io/pgvector:0.8-pg16` under CNPG | Same major and same pgvector minor line, so the planner behaviour the design doc measured still holds. CNPG additionally needs `vector` created at bootstrap — see [The database](#the-database) | +| healthcheck | none — distroless has no shell | HTTP probes | The compose file notes the probes are HTTP and "for you rather than for Compose" | +| Superuser | `POSTGRES_USER=discovery` **is** the superuser | the service owns its own database and is not superuser | This is why the `vector` extension problem appears only in the cluster: in compose the migration creates it as superuser without anyone noticing it needed to be one. The cluster creates it at bootstrap instead | + +## Validation + +```bash +../../scripts/lint-charts.sh +``` + +Two `ci/*-values.yaml` files render as separate cases, deliberately covering +opposite branches: `lint-values.yaml` is the assembled DSN with a fetched spec, +`url-secret-values.yaml` is the whole-DSN Secret with a ConfigMap spec, plus +autoscaling-adjacent extras, Ingress and a PDB. A chart that only ever renders +its defaults has untested branches that fail on the day someone uses them. + +Render-time guardrails, all of which name the value and the reason: + +- `image.repository` empty +- `app.networkId` empty +- neither `database.urlSecret.name` nor `database.passwordSecret.name` +- `database.host` empty while the DSN is assembled +- neither `becknSpec.url` nor `becknSpec.existingConfigMap` +- `embeddings.provider` other than `noop` with no `embeddings.endpoint` +- `otel.exporter` other than `none` with no `otel.endpoint` +- `resources` empty (`oan-common`) +- a probe with no handler or with two (`oan-common`) +- a PodDisruptionBudget alongside a single replica diff --git a/charts/discovery/ci/lint-values.yaml b/charts/discovery/ci/lint-values.yaml new file mode 100644 index 0000000..5491ee8 --- /dev/null +++ b/charts/discovery/ci/lint-values.yaml @@ -0,0 +1,20 @@ +# Minimum values that let `helm template` run in CI. Dummy references only - +# see examples/ for real per-environment configuration. +# +# Covers the assembled-DSN path (database.passwordSecret). The whole-DSN path +# is covered by ci/url-secret-values.yaml. +image: + registry: ghcr.io + repository: openagrinet/discovery-service + tag: "0.1.0" + +app: + networkId: local-network + +database: + host: discovery-db-rw + passwordSecret: + name: discovery-db-app + +becknSpec: + url: https://example.invalid/beckn.yaml diff --git a/charts/discovery/ci/url-secret-values.yaml b/charts/discovery/ci/url-secret-values.yaml new file mode 100644 index 0000000..01508db --- /dev/null +++ b/charts/discovery/ci/url-secret-values.yaml @@ -0,0 +1,53 @@ +# The other half of the matrix: the DSN comes whole from a Secret key, the spec +# from a ConfigMap rather than a URL, and the optional resources are all on. +# +# Rendering this is what keeps the alternative paths honest - a chart that only +# ever renders its defaults has untested branches that fail on the day someone +# uses them. +image: + registry: ghcr.io + repository: openagrinet/discovery-service + tag: "0.1.0" + +replicaCount: 2 + +app: + networkId: local-network + +database: + urlSecret: + name: discovery-db-app + key: uri + autoMigrate: true + +becknSpec: + existingConfigMap: discovery-beckn-spec + +embeddings: + provider: ollama + endpoint: http://ollama:11434 + +otel: + exporter: otlp + endpoint: http://otel-collector:4317 + +replicationTargets: + - https://peer-a.example.invalid/publish + - https://peer-b.example.invalid/publish + +ingress: + enabled: true + className: alb + hosts: + - host: discovery.example.invalid + paths: + - path: / + pathType: Prefix + +podDisruptionBudget: + enabled: true + minAvailable: 1 + +envConfig: + RATE_LIMIT_RPS: "50" + RATE_LIMIT_BURST: "100" diff --git a/charts/discovery/examples/discovery.dev.yaml b/charts/discovery/examples/discovery.dev.yaml new file mode 100644 index 0000000..48a0f8e --- /dev/null +++ b/charts/discovery/examples/discovery.dev.yaml @@ -0,0 +1,85 @@ +# Example: the OAN discover-and-publish service, dev environment. +# +# helm install discovery charts/discovery \ +# -n oan-discovery -f charts/discovery/examples/discovery.dev.yaml +# +# Install the database FIRST, and with an image that has pgvector - see +# README.md, "The database". Starting before the database is up is not fatal - +# the pod crashloops until it can connect, and Kubernetes' own backoff is the +# retry - but a database without pgvector never becomes reachable in the sense +# that matters, because the migration fails on the missing extension. + +# No fullnameOverride needed: the chart is named "discovery", so a release +# named "discovery" already produces Service/discovery. + +# TODO: no image is published for discovery-service yet - CI builds and scans +# one but pushes nothing. Fill this in with whatever the first published tag is; +# the render fails until then, on purpose. +image: + registry: ghcr.io + repository: openagrinet/discovery-service + tag: "0.1.0" + +app: + # The network this deployment serves. It fills an empty + # publishDirectives.visibleTo, so a catalog published without one is visible + # to this network and no other - which makes this the single most consequential + # value in the file. + networkId: mahavistar + +logLevel: debug + +database: + # CNPG generates Secret/-app with a ready-made `uri` key, so there is + # nothing to assemble and no password escaping to get wrong. The DSN carries + # host, port, database and user, so none of those are set here - a second copy + # would be one more thing to keep true. + urlSecret: + name: discovery-db-app + key: uri + + # Dev migrates on boot. Keep replicaCount at 1 while this is true: the + # migration runs in-process, so several pods starting at once race to apply + # the same one. + autoMigrate: true + + # Two retrieval modes today, one in-flight discover per mode. 32 is generous + # for dev and leaves room for the third mode when semantic search lands. + maxConns: 32 + minConns: 4 + +# Fetched at boot and cached in an emptyDir. For a cluster with no egress, or +# for a reproducible boot, create the ConfigMap instead: +# kubectl -n oan-discovery create configmap discovery-beckn-spec \ +# --from-file=beckn.yaml=/tests/testdata/beckn-v2.0.0.yaml +# and set existingConfigMap below. Note this URL tracks `main` and so pins +# nothing. +becknSpec: + url: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/heads/main/api/v2.0.0/beckn.yaml + existingConfigMap: "" + +# Semantic search stays off until an Ollama deployment exists (A5). Discover +# still answers; it degrades the semantic mode and names it in X-Beckn-Degraded. +embeddings: + provider: noop + +otel: + exporter: none + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +ingress: + enabled: false + +# `helm test discovery -n oan-discovery` checks /readyz. +tests: + enabled: true + +commonLabels: + oan.in/environment: dev diff --git a/charts/discovery/examples/discovery.prod.yaml b/charts/discovery/examples/discovery.prod.yaml new file mode 100644 index 0000000..378d8fb --- /dev/null +++ b/charts/discovery/examples/discovery.prod.yaml @@ -0,0 +1,122 @@ +# Example: the OAN discover-and-publish service, production. +# +# helm install discovery charts/discovery \ +# -n oan-discovery -f charts/discovery/examples/discovery.prod.yaml +# +# NOT INSTALLABLE AS-IS. image.repository is empty and the render fails until +# it is filled in - see below. + +# --------------------------------------------------------------------------- +# TODO: the image OAN deploys in production is not decided. Prefer a DIGEST +# over a tag: a tag can be repointed at different bits, a digest cannot, which +# is what makes a rollback land on the same image it did before. +# +# image: +# registry: ghcr.io +# repository: openagrinet/discovery-service +# digest: sha256:... +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: "" + digest: "" + +replicaCount: 3 + +app: + networkId: mahavistar + +logLevel: info + +database: + urlSecret: + name: discovery-db-app + key: uri + + # OFF in production. Migrating is a step someone decides to take, on a + # release that was chosen for it - not something that happens because a pod + # was rescheduled at 3am. To migrate: + # + # helm upgrade discovery -n oan-discovery --reuse-values \ + # --set database.autoMigrate=true --set replicaCount=1 + # # verify, then upgrade back + autoMigrate: false + + # 3 replicas x 48 = 144 connections at full stretch. Check the cluster's + # max_connections before raising either number, and remember the autoscaler + # multiplies this by maxReplicas rather than by replicaCount. + maxConns: 48 + minConns: 8 + +# A ConfigMap rather than a fetch: production should not have its boot depend +# on GitHub being reachable, and the upstream URL tracks `main`, so a fetch +# pins nothing. Create it from a reviewed copy of the document: +# kubectl -n oan-discovery create configmap discovery-beckn-spec \ +# --from-file=beckn.yaml=beckn-v2.0.0.yaml +becknSpec: + url: "" + existingConfigMap: discovery-beckn-spec + key: beckn.yaml + +embeddings: + provider: noop + +# TODO: point at the cluster's collector once one is deployed. +otel: + exporter: none + endpoint: "" + +resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 1Gi + +ingress: + enabled: true + className: alb + annotations: + alb.ingress.kubernetes.io/scheme: internal + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/healthcheck-path: /readyz + hosts: + - host: discovery.oan.in + paths: + - path: / + pathType: Prefix + tls: [] + +# Safe with 3 replicas: at most one pod is disrupted at a time, so a node drain +# proceeds without taking the service below two. +podDisruptionBudget: + enabled: true + maxUnavailable: 1 + minAvailable: "" + +# Spread across nodes, so one node's loss is not the whole service. `preferred` +# rather than `required`: a hard rule leaves pods Pending when the cluster has +# fewer nodes than replicas, which turns a scale-up into an outage. +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: discovery + app.kubernetes.io/instance: discovery + +# Left OFF deliberately. Enabling it means capacity is decided by CPU, and the +# ceiling it puts on the database is maxReplicas x database.maxConns - raise +# them together, or not at all. +autoscaling: + enabled: false + +tests: + enabled: true + +commonLabels: + oan.in/environment: prod diff --git a/charts/discovery/templates/NOTES.txt b/charts/discovery/templates/NOTES.txt new file mode 100644 index 0000000..36ae0a4 --- /dev/null +++ b/charts/discovery/templates/NOTES.txt @@ -0,0 +1,59 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "discovery.fullname" . }} ({{ if .Values.autoscaling.enabled }}autoscaled {{ .Values.autoscaling.minReplicas }}-{{ .Values.autoscaling.maxReplicas }}{{ else }}{{ .Values.replicaCount }} replica(s){{ end }}) + - Service/{{ include "discovery.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "discovery.envConfigMapName" . }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "discovery.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "discovery.fullname" . }} +{{- end }} +{{- if .Values.podDisruptionBudget.enabled }} + - PodDisruptionBudget/{{ include "discovery.fullname" . }} +{{- end }} + +Wired to: + network: {{ .Values.app.networkId }} + database: {{ if .Values.database.urlSecret.name }}DSN from Secret/{{ .Values.database.urlSecret.name }} key {{ .Values.database.urlSecret.key }}{{ else }}{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }} (as {{ .Values.database.user }}){{ end }} + migrations: {{ if .Values.database.autoMigrate }}APPLIED ON BOOT{{ else }}not applied - DATABASE_AUTO_MIGRATE is false{{ end }} + beckn spec: {{ if .Values.becknSpec.url }}{{ .Values.becknSpec.url }}{{ else }}no URL - cache only{{ end }} + cache {{ if .Values.becknSpec.existingConfigMap }}from ConfigMap/{{ .Values.becknSpec.existingConfigMap }}{{ else }}in an emptyDir (refetched on every reschedule){{ end }} at {{ include "discovery.specCachePath" . }} + embeddings: {{ .Values.embeddings.provider }}{{ if eq .Values.embeddings.provider "noop" }} - semantic search is OFF, and discover says so in X-Beckn-Degraded{{ end }} + +In-cluster endpoint: + http://{{ include "discovery.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +Check it: + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "discovery.fullname" . }} + helm test {{ .Release.Name }} -n {{ .Release.Namespace }} + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "discovery.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl http://localhost:{{ .Values.service.port }}/healthz # liveness - depends on nothing + curl http://localhost:{{ .Values.service.port }}/readyz # readiness - 503 when PostgreSQL is unreachable + +{{ if not .Values.database.autoMigrate -}} +Migrations are OFF. On a database that has never been migrated, every request +fails against missing tables while /healthz still answers 200. To migrate: + + helm upgrade {{ .Release.Name }} -n {{ .Release.Namespace }} --reuse-values \ + --set database.autoMigrate=true --set replicaCount=1 + # then turn it back off once the rollout is complete + +Scale to one replica first: the migration runs in-process on boot, so several +pods starting at once race to apply it. +{{ end -}} + +If the pods crashloop before serving, check these in order - the first two look +nothing like their cause in the logs: + 1. "permission denied to create extension vector" means the database was not + bootstrapped with it. `vector` is not a trusted extension, so the owner + this service connects as cannot create it - the cluster must, at bootstrap, + as superuser. See charts/postgresql-cnpg/examples/discovery-db.dev.yaml. + "could not open extension control file" instead means the image has no + pgvector at all; no stock CNPG operand image does. + 2. "load the validation spec" means neither the fetch nor the cache produced + a Beckn document. With becknSpec.url set and no ConfigMap, the pod needs + egress to that URL. + 3. A key in `envConfig` matching no config field fails the boot by design, so + a typo there stops the service rather than being ignored. diff --git a/charts/discovery/templates/_helpers.tpl b/charts/discovery/templates/_helpers.tpl new file mode 100644 index 0000000..fc5fc4f --- /dev/null +++ b/charts/discovery/templates/_helpers.tpl @@ -0,0 +1,178 @@ +{{/* +# ============================================================================ +# DISCOVERY SERVICE CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the database, +# Beckn specification and environment wiring this service needs. +# ============================================================================ +*/}} + +{{- define "discovery.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "discovery.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "discovery.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "discovery.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "discovery.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "discovery.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "discovery.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +Absolute path of the Beckn spec cache, VALIDATION_SPEC_CACHE_PATH. + +Always absolute and always set explicitly, rather than left to the service's +own default of ".cache/beckn/beckn.yaml": that one is relative to the working +directory, which makes the path a fact about the image rather than about this +chart, and the volume mount below has to name a directory that agrees with it. +*/}} +{{- define "discovery.specCacheDir" -}} +{{- .Values.becknSpec.cacheDir | trimSuffix "/" -}} +{{- end }} + +{{- define "discovery.specCachePath" -}} +{{- printf "%s/%s" (include "discovery.specCacheDir" .) .Values.becknSpec.key -}} +{{- end }} + +{{/* +Emits "true" when the spec cache is backed by a ConfigMap rather than by an +emptyDir the fetched document is written into. +*/}} +{{- define "discovery.specFromConfigMap" -}} +{{- if .Values.becknSpec.existingConfigMap -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +The DSN, when this chart assembles one. Empty when database.urlSecret is set, +because then the whole DSN comes from that Secret instead. + +The password is NOT interpolated here - it is referenced as $(DATABASE_PASSWORD) +and expanded by the kubelet against the secretKeyRef env var defined just above +it in the container spec. That keeps the password out of the rendered manifest, +out of `helm get values`, and out of anything that logs a template. + +Because that expansion is textual, a password containing a URL delimiter +(@ : / ? # %) must already be percent-encoded in the Secret. urlSecret avoids +the question entirely, which is why it is the documented default. +*/}} +{{- define "discovery.databaseURL" -}} +{{- $db := .Values.database -}} +{{- printf "postgres://%s:$(DATABASE_PASSWORD)@%s:%v/%s?sslmode=%s" $db.user $db.host $db.port $db.name $db.sslMode -}} +{{- end }} + +{{/* +Everything the service needs that this chart derives from its structured +values, as container env entries. These take precedence over the envConfig +ConfigMap injected with envFrom, which is what makes "derived here, overridable +there" safe rather than ambiguous. + +Every guardrail below refuses to render something the service would refuse to +boot on. The failure is the same either way; the difference is that this one +names the value and happens before anything is applied to the cluster. +*/}} +{{- define "discovery.env" -}} +{{- $db := .Values.database -}} +{{- $spec := .Values.becknSpec -}} +{{- if not .Values.app.networkId }} +{{- fail (printf "%s: app.networkId is required - it is APP_NETWORK_ID, which the boot refuses without, and it decides who a catalog published with no explicit visibleTo is visible to. Set the network this deployment serves, e.g. mahavistar." .Chart.Name) }} +{{- end }} +{{- if and (not $db.urlSecret.name) (not $db.passwordSecret.name) }} +{{- fail (printf "%s: the database DSN is unset. Set database.urlSecret.name (preferred - CNPG writes a ready-made `uri` key into Secret/-app), or database.passwordSecret.name together with database.host/name/user to have the chart assemble one. This chart renders no Secrets." .Chart.Name) }} +{{- end }} +{{- if and (not $db.urlSecret.name) (not $db.host) }} +{{- fail (printf "%s: database.host is required when the DSN is assembled - point it at the PostgreSQL primary service, e.g. discovery-db-rw" .Chart.Name) }} +{{- end }} +{{- if and (not $spec.url) (not $spec.existingConfigMap) }} +{{- fail (printf "%s: set becknSpec.url or becknSpec.existingConfigMap. The service loads the Beckn document before it serves and refuses to start without it, and with neither set there is nothing to fetch and nothing cached to fall back to." .Chart.Name) }} +{{- end }} +{{- if not $spec.key }} +{{- fail (printf "%s: becknSpec.key is required - it is both the key read from the ConfigMap and the filename the cache is written under" .Chart.Name) }} +{{- end }} +{{- if $db.urlSecret.name -}} +- name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ $db.urlSecret.name }} + key: {{ $db.urlSecret.key }} +{{- else }} +{{- /* Defined FIRST: the kubelet expands $(VAR) only against variables that + precede it in this list. */ -}} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $db.passwordSecret.name }} + key: {{ $db.passwordSecret.key }} +- name: DATABASE_URL + value: {{ include "discovery.databaseURL" . | quote }} +{{- end }} +- name: DATABASE_AUTO_MIGRATE + value: {{ $db.autoMigrate | quote }} +- name: DATABASE_MAX_CONNS + value: {{ $db.maxConns | quote }} +- name: DATABASE_MIN_CONNS + value: {{ $db.minConns | quote }} +- name: APP_NETWORK_ID + value: {{ .Values.app.networkId | quote }} +- name: APP_DEFAULT_TIMEZONE + value: {{ .Values.app.defaultTimezone | quote }} +{{- /* Derived from the port the container actually publishes, so the two + cannot drift into a Service that routes to a port nothing listens on. */}} +- name: SERVER_PORT + value: {{ .Values.service.targetPort | quote }} +- name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} +- name: VALIDATION_SPEC_CACHE_PATH + value: {{ include "discovery.specCachePath" . | quote }} +{{- with $spec.url }} +- name: VALIDATION_SPEC_URL + value: {{ . | quote }} +{{- end }} +- name: EMBEDDING_PROVIDER + value: {{ .Values.embeddings.provider | quote }} +{{- if ne .Values.embeddings.provider "noop" }} +- name: EMBEDDING_MODEL + value: {{ .Values.embeddings.model | quote }} +- name: EMBEDDING_DIMENSIONS + value: {{ .Values.embeddings.dimensions | quote }} +{{- if not .Values.embeddings.endpoint }} +{{- fail (printf "%s: embeddings.provider is %q, so embeddings.endpoint is required - a provider with nowhere to call fails at the first publish, not at boot" .Chart.Name .Values.embeddings.provider) }} +{{- end }} +- name: EMBEDDING_ENDPOINT + value: {{ .Values.embeddings.endpoint | quote }} +{{- end }} +- name: OTEL_EXPORTER + value: {{ .Values.otel.exporter | quote }} +{{- if ne .Values.otel.exporter "none" }} +{{- if not .Values.otel.endpoint }} +{{- fail (printf "%s: otel.exporter is %q, so otel.endpoint is required - set it to the collector, e.g. http://otel-collector:4317" .Chart.Name .Values.otel.exporter) }} +{{- end }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} +{{- with .Values.replicationTargets }} +- name: REPLICATION_TARGETS + value: {{ join "," . | quote }} +{{- end }} +{{- with (include "oan-common.env" . | trim) }} +{{ . }} +{{- end }} +{{- end }} diff --git a/charts/discovery/templates/configmap.yaml b/charts/discovery/templates/configmap.yaml new file mode 100644 index 0000000..eea3b40 --- /dev/null +++ b/charts/discovery/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "discovery.envConfigMapName" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with (include "oan-common.envConfigMapData" . | trim) }} +data: + {{- . | nindent 2 }} +{{- else }} +data: {} +{{- end }} diff --git a/charts/discovery/templates/deployment.yaml b/charts/discovery/templates/deployment.yaml new file mode 100644 index 0000000..82b37b3 --- /dev/null +++ b/charts/discovery/templates/deployment.yaml @@ -0,0 +1,111 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "discovery.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "discovery.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "discovery.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "discovery.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- include "discovery.env" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "discovery.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + {{- /* + The Beckn spec cache. Mounted either way, and not only for the + air-gapped case: with readOnlyRootFilesystem the fetch path has + nowhere to write its cache without a volume here, and /app is not + writable by uid 65532 in any case. + */}} + - name: beckn-spec + mountPath: {{ include "discovery.specCacheDir" . }} + readOnly: {{ if include "discovery.specFromConfigMap" . }}true{{ else }}false{{ end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: beckn-spec + {{- if include "discovery.specFromConfigMap" . }} + configMap: + name: {{ .Values.becknSpec.existingConfigMap }} + items: + - key: {{ .Values.becknSpec.key }} + path: {{ .Values.becknSpec.key }} + {{- else }} + emptyDir: + sizeLimit: 16Mi + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/discovery/templates/hpa.yaml b/charts/discovery/templates/hpa.yaml new file mode 100644 index 0000000..9687662 --- /dev/null +++ b/charts/discovery/templates/hpa.yaml @@ -0,0 +1,54 @@ +{{- if .Values.autoscaling.enabled }} +{{/* +The service is stateless - it holds no session state and writes everything to +PostgreSQL - so it scales horizontally. Two things to keep in mind: + + - Every replica opens its own pool of up to database.maxConns connections, so + the ceiling this chart puts on the server is maxReplicas x maxConns. Raise + the two together, against the server's `max_connections`, not separately. + - Autoscaling takes over `replicas`, so replicaCount is ignored once this is + enabled. The Deployment deliberately omits `replicas` in that case, or the + two would fight on every reconcile. + +Do NOT enable this for the install that migrates: DATABASE_AUTO_MIGRATE with +several pods starting at once has them race to apply the same migration. +*/}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "discovery.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- with .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/discovery/templates/ingress.yaml b/charts/discovery/templates/ingress.yaml new file mode 100644 index 0000000..c6c9dfd --- /dev/null +++ b/charts/discovery/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "discovery.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/discovery/templates/poddisruptionbudget.yaml b/charts/discovery/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..1fa8a56 --- /dev/null +++ b/charts/discovery/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Caps how many pods can be voluntarily disrupted at once - node drains, cluster +upgrades, autoscaler scale-down. + +With replicaCount: 1 a PDB of minAvailable: 1 blocks drains entirely, because +evicting the only pod would breach it. That is why this is disabled by default +and should be enabled together with replicaCount >= 2. +*/}} +{{- if and (eq (int .Values.replicaCount) 1) (not .Values.podDisruptionBudget.allowSingleReplica) }} +{{- fail (printf "%s: podDisruptionBudget is enabled with replicaCount 1, which blocks node drains entirely. Raise replicaCount to 2+, or set podDisruptionBudget.allowSingleReplica=true if you accept that." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "discovery.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/discovery/templates/service.yaml b/charts/discovery/templates/service.yaml new file mode 100644 index 0000000..37a301b --- /dev/null +++ b/charts/discovery/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "discovery.selectorLabels" . | nindent 4 }} diff --git a/charts/discovery/templates/serviceaccount.yaml b/charts/discovery/templates/serviceaccount.yaml new file mode 100644 index 0000000..90c4e8c --- /dev/null +++ b/charts/discovery/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "discovery.serviceAccountName" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/discovery/templates/tests/test-connection.yaml b/charts/discovery/templates/tests/test-connection.yaml new file mode 100644 index 0000000..7492d2f --- /dev/null +++ b/charts/discovery/templates/tests/test-connection.yaml @@ -0,0 +1,40 @@ +{{- if .Values.tests.enabled }} +{{/* +`helm test` check: the service answers on /readyz. + +/readyz rather than /healthz on purpose. /healthz depends on nothing and stays +200 with the database down, so a test against it would pass on a deployment +that cannot serve a single discover. /readyz pings the datastore, which is the +question worth asking after an install. + +Deliberately does not POST a discover: that needs a published catalog to find, +and a failure would be ambiguous between "the service is broken" and "nothing +has been published yet". +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "discovery.fullname" . }}-test-connection + labels: + {{- include "discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: test + image: {{ printf "%s/%s:%s" .Values.tests.image.registry .Values.tests.image.repository .Values.tests.image.tag | quote }} + command: + - /bin/sh + - -c + - | + set -e + url="http://{{ include "discovery.fullname" . }}:{{ .Values.service.port }}/readyz" + echo "GET $url" + wget -q -O- --timeout=10 "$url" + echo "" + echo "OK: discovery is ready - it is serving and it can reach PostgreSQL" + resources: + {{- toYaml .Values.tests.resources | nindent 8 }} +{{- end }} diff --git a/charts/discovery/values.yaml b/charts/discovery/values.yaml new file mode 100644 index 0000000..90c698f --- /dev/null +++ b/charts/discovery/values.yaml @@ -0,0 +1,416 @@ +# ============================================================================ +# discovery - default values +# +# The OAN Beckn v2.0.0 discover-and-publish service. Defaults mirror the +# verified compose stack (discovery-service/docker-compose.yml), with the two +# settings that file marks as local-only put back to their deployment values: +# DATABASE_AUTO_MIGRATE and the rate limiter. +# +# Depends on one thing already running: +# - PostgreSQL, WITH the `vector` and `pg_trgm` extensions -> database.* +# +# Minimum you must set: +# - image.repository +# - app.networkId +# - database.host, and either database.urlSecret.name or +# database.passwordSecret.name +# - becknSpec.url or becknSpec.existingConfigMap +# +# The render fails, with the reason, if any of those is missing. That is +# deliberate: every one of them is something the service refuses to boot +# without, and a pod that CrashLoopBackOffs reports the problem far later and +# far less clearly than `helm template` does. +# ============================================================================ + +replicaCount: 1 + +# --------------------------------------------------------------------------- +# Image +# +# repository is EMPTY on purpose. discovery-service's CI builds an image and +# scans it but pushes it nowhere, so there is no tag to default to; the compose +# stack builds from the working tree. The render fails while this is empty +# rather than producing "ghcr.io/:0.1.0", which Helm and the API server both +# accept and which only surfaces later as an ImagePullBackOff. +# +# The runtime is gcr.io/distroless/static-debian12:nonroot - no shell, no +# package manager, uid 65532. That is what makes the security context defaults +# below safe to leave on. +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: "" + tag: "" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: false + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + annotations: {} + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: discovery.local + paths: + - path: / + pathType: Prefix + tls: [] + +# --------------------------------------------------------------------------- +# Application +# --------------------------------------------------------------------------- +app: + # APP_NETWORK_ID. REQUIRED - the boot refuses without it, so the render does + # too. It fills an empty publishDirectives.visibleTo (C8), which means every + # catalog published without one is visible to this network and no other. + # There is no sensible default: "mahavistar", "bharatvistar" and the rest are + # different networks, and guessing wrong publishes to the wrong audience. + networkId: "" + # APP_DEFAULT_TIMEZONE. Validated with time.LoadLocation at startup - a typo + # fails the boot rather than silently shifting every daily validity window. + defaultTimezone: Asia/Kolkata + +# LOG_LEVEL +logLevel: info + +# --------------------------------------------------------------------------- +# Database +# +# The service needs PostgreSQL with two extensions its first migration creates: +# +# CREATE EXTENSION IF NOT EXISTS vector; -- pgvector, for the embedding column and its HNSW index +# CREATE EXTENSION IF NOT EXISTS pg_trgm; -- contrib, ships with every ordinary PostgreSQL build +# +# pg_trgm is contrib and is present in the CloudNativePG operand images. +# pgvector is NOT: the stock CNPG image cannot create the `vector` extension, +# and the migration - and therefore the boot - fails on it. Point +# postgresql-cnpg's `image.*` at an operand image that bundles pgvector before +# installing this chart. See README.md, "The database". +# +# Two ways to supply the DSN, and exactly one of them must be set: +# +# urlSecret one Secret key holding the whole DATABASE_URL. PREFERRED: +# CNPG already writes a ready-made `uri` key into the app +# Secret it generates, so there is nothing to assemble and no +# escaping to get wrong. +# +# host/port/name/user + passwordSecret +# the DSN is assembled from these, with the password injected +# through Kubernetes' own $(VAR) expansion so it stays in the +# Secret and never lands in a ConfigMap. Note the caveat: the +# password is substituted verbatim into a URL, so a password +# containing @ : / ? # or % must already be percent-encoded +# in the Secret. +# +# urlSecret makes host/port/name/user/sslMode unread - the DSN carries all of +# it - so leave them empty rather than setting a second copy that nothing +# consumes and nothing keeps true. +# --------------------------------------------------------------------------- +database: + # PostgreSQL primary service, e.g. "discovery-db-rw" + host: "" + port: 5432 + name: discovery + user: discovery + # disable | allow | prefer | require | verify-ca | verify-full. Only used + # when the DSN is assembled. + sslMode: prefer + # Whole-DSN Secret. Wins over the assembled form when set. + urlSecret: + name: "" + # CNPG writes `uri` into Secret/-app + key: uri + # Password Secret, for the assembled form. + passwordSecret: + name: "" + key: password + + # DATABASE_AUTO_MIGRATE. The migrations are compiled into the binary, so this + # needs no sidecar and no mounted directory - the service applies them on + # boot and treats "already at the latest version" as success. + # + # False here, and true in compose, is not a discrepancy: migrating is a step + # someone decides to take. Turn it on for the install that is meant to + # migrate, and note that with replicaCount > 1 several pods race to do it - + # scale to 1, migrate, scale back. + autoMigrate: false + + # Sized by the concurrency model, not guessed: discover runs its retrieval + # modes concurrently (A2), so one in-flight discover holds as many + # connections as it has enabled modes. + # + # maxConns >= (enabled modes) x (expected in-flight discovers) + # + # Two modes today, three once semantic lands. Multiply by replicaCount and + # keep the total under the server's max_connections. minConns is a warm-start + # knob only - idle backends cost the server memory to save a handshake. + maxConns: 32 + minConns: 4 + +# --------------------------------------------------------------------------- +# Beckn specification +# +# The service loads the Beckn v2.0.0 document BEFORE it serves and refuses to +# start without it. The check is unconditional: turning L1 validation off does +# not remove the requirement. +# +# It tries `url` first and falls back to the on-disk cache, so at least one of +# `url` and `existingConfigMap` must be set or the boot cannot succeed - the +# render fails when both are empty. +# +# The document is deliberately NOT shipped in this chart, for the same reason +# the Dockerfile does not bake it into the image: a copy here is a second +# source of truth that ages independently of the protocol. +# --------------------------------------------------------------------------- +becknSpec: + # VALIDATION_SPEC_URL. Upstream is + # https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/heads/main/api/v2.0.0/beckn.yaml + # which tracks `main` and therefore pins nothing. Prefer a pinned URL, or the + # ConfigMap below, for a reproducible boot. + url: "" + # A ConfigMap holding the document, mounted as the cache. Set this for an + # air-gapped deploy, or alongside `url` as the fallback when the fetch fails. + # kubectl -n create configmap discovery-beckn-spec \ + # --from-file=beckn.yaml=tests/testdata/beckn-v2.0.0.yaml + existingConfigMap: "" + # Key inside that ConfigMap, and the filename it is mounted under. + key: beckn.yaml + # Directory the cache lives in. VALIDATION_SPEC_CACHE_PATH is this plus key. + # An emptyDir is mounted here when no ConfigMap is supplied, so the fetched + # document can be cached - /app is not writable by uid 65532. + cacheDir: /app/.cache/beckn + +# --------------------------------------------------------------------------- +# Embeddings +# +# Semantic search is deferred (A5): the column, the HNSW index and the Embedder +# seam all ship, the provider does not. `noop` is the reviewed default and the +# one Phase 1 runs on - a discover request simply degrades and says so in +# X-Beckn-Degraded (C11). +# +# Set provider: ollama and an endpoint once an Ollama deployment exists. +# dimensions must match the column the migration created; changing it on a +# populated database is a reindex, not a config change. +# --------------------------------------------------------------------------- +embeddings: + provider: noop + model: nomic-embed-text + endpoint: "" + dimensions: 768 + +# --------------------------------------------------------------------------- +# OpenTelemetry +# +# `none` boots without a collector (T2). Set exporter: otlp and an endpoint to +# export. +# --------------------------------------------------------------------------- +otel: + exporter: none + endpoint: "" + +# REPLICATION_TARGETS - peer discovery endpoints a publish is fanned out to. +# Rendered as the comma-separated list the service parses. +replicationTargets: [] + +resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + +# --------------------------------------------------------------------------- +# Probes +# +# /healthz and /readyz answer different questions, and the split is load-bearing +# (src/app/router.go): /healthz depends on nothing and stays 200 with the +# database down, /readyz pings the database and returns 503 when it cannot. +# Pointing liveness at /readyz would restart every pod during a database blip - +# turning a recoverable outage into a crashloop across the whole Deployment. +# +# There is no `exec` option and no compose-style healthcheck: the runtime stage +# is distroless/static, which has no shell for one to run. +# --------------------------------------------------------------------------- +startupProbe: + enabled: true + httpGet: + path: /healthz + port: http + # Generous: the boot fetches and compiles the Beckn document, and may apply + # migrations, before it listens. + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + +livenessProbe: + enabled: true + httpGet: + path: /healthz + port: http + initialDelaySeconds: 0 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + +readinessProbe: + enabled: true + httpGet: + path: /readyz + port: http + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# --------------------------------------------------------------------------- +# Security contexts +# +# On by default here, unlike the other OAN service charts. The image is +# distroless/static running as uid 65532 with a fully static binary: there is +# no shell to escalate into, and the only path the process writes is the spec +# cache, which is a mounted volume. readOnlyRootFilesystem therefore costs +# nothing - and if you add anything that needs scratch space, add an emptyDir +# through extraVolumes rather than turning this off. +# --------------------------------------------------------------------------- +podSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + +securityContext: + enabled: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# Extra init containers. Rendered verbatim. +extraInitContainers: [] + +# Extra volumes and mounts, rendered verbatim. +extraVolumes: [] +extraVolumeMounts: [] + +# --------------------------------------------------------------------------- +# PodDisruptionBudget +# +# Caps voluntary disruptions - node drains, cluster upgrades, autoscaler +# scale-down. Disabled by default because with a single replica a PDB of +# minAvailable: 1 blocks drains entirely; the render fails if you enable it +# anyway without acknowledging that. +# --------------------------------------------------------------------------- +podDisruptionBudget: + enabled: false + minAvailable: 1 + # Set one or the other, not both. + maxUnavailable: "" + # Enable a PDB with a single replica anyway, accepting that node drains block. + allowSingleReplica: false + +# --------------------------------------------------------------------------- +# Autoscaling +# +# The service is stateless - everything is in PostgreSQL - so it scales +# horizontally. Every replica opens its own pool of up to database.maxConns +# connections, so raise maxReplicas and maxConns together against the server's +# max_connections, not separately. When enabled, replicaCount is ignored. +# --------------------------------------------------------------------------- +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + # Set to a number to also scale on memory. + targetMemoryUtilizationPercentage: "" + # Scale-up/scale-down tuning, rendered verbatim into spec.behavior. + behavior: {} + +# --------------------------------------------------------------------------- +# `helm test` checks +# +# helm test -n +# +# Runs after install to confirm the service actually answers - which is the +# difference between "the pod is running" and "the deployment works". It checks +# /readyz rather than /healthz, because /healthz is 200 with the database down +# and would pass on a deployment that cannot serve a single request. +# --------------------------------------------------------------------------- +tests: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + +# --------------------------------------------------------------------------- +# ENVIRONMENT CONFIGURATION +# +# Rendered into a ConfigMap and injected with envFrom. EMPTY on purpose. +# +# The image already carries config/common.yaml - the project's reviewed +# defaults for search sizing, validation, auth and the rest - and the +# environment layer sits above it. Copying those values here would create a +# second copy of decisions this chart did not make, and it is the second copy +# that rots. Everything the chart genuinely owns is derived from the structured +# values above instead, so nothing here can drift from them. +# +# Use this to override a common.yaml default for one deployment. The full list +# is in README.md; the ones most likely to be wanted: +# +# SEARCH_DEFAULT_PAGE_SIZE, SEARCH_MAX_PAGE_SIZE, SEARCH_MAX_RADIUS_METERS +# SEARCH_MAX_CANDIDATES_PER_MODE, SEARCH_READ_DEADLINE +# SEARCH_FAIL_ON_UNAVAILABLE_MODE +# RATE_LIMIT_RPS, RATE_LIMIT_BURST (defaults 20 / 40; burst >= rps > 0) +# SERVER_SHUTDOWN_TIMEOUT, SERVER_MAX_REQUEST_BODY_BYTES +# VALIDATION_ENABLE_L1_SCHEMA, GEO_RESOLUTION_CELLS +# +# A key that matches no Config field fails the boot, so a typo here cannot +# silently do nothing. +# +# NEVER put secrets here. This is a ConfigMap. +# --------------------------------------------------------------------------- +envConfig: {} + +# Extra env vars from specific Secret keys, beyond DATABASE_URL. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} diff --git a/charts/registry/.helmignore b/charts/registry/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/registry/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/registry/CHANGELOG.md b/charts/registry/CHANGELOG.md new file mode 100644 index 0000000..3aaa256 --- /dev/null +++ b/charts/registry/CHANGELOG.md @@ -0,0 +1,74 @@ +# Changelog + +All notable changes to the `registry` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Changed +- `image.repository` now defaults to empty rather than + `sunbird-rc/sunbird-rc-core`. The image OAN will deploy is not settled, and a + default was reading as a decision that had not been made. The render fails + while it is empty, so this cannot be missed. The dev example sets the + compose-verified image explicitly; the production example is a documented TODO. + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Configuration ported from the verified +`registry/docker-compose.yml` stack, with the deployment shape taken from +Sunbird's own Helm charts (`deploy-as-code/helm/v2`). + +### Added +- Deployment, Service, env ConfigMap, schemas ConfigMap, ServiceAccount, and + optional Ingress and ESO ExternalSecret, built on the `oan-common` library. +- All 32 registry environment variables from the compose stack, verified at + parity. Database and Keycloak variables are derived from structured values + rather than restated, so `OAUTH2_RESOURCES_0_URI` cannot drift from + `keycloak.url` and `connectionInfo_uri` cannot drift from `database.*`. +- Entity schemas shipped in `files/schemas/` and rendered into a ConfigMap + mounted read-only, with `existingConfigMap` and `inline` overrides. A + `checksum/schemas` annotation rolls the pod when a schema changes. +- Render-time validation of `database.host`, `database.passwordSecret.name`, + `keycloak.url`, `keycloak.adminClientSecret.name`, and + `defaultUserPasswordSecret.name` when `keycloakUserSetPassword` is true. + Also fails when no schema file matches, since a registry with no entity + definitions serves nothing. +- Every optional Sunbird RC subsystem off by default, matching compose: + encryption, events, idgen, claims, DIDs, signatures, certificates, file + storage, notifications, async and webhooks. + +- Init containers that wait for the database and for Keycloak's realm endpoint. + The Keycloak check is HTTP against the realm rather than TCP against the port, + because the realm is imported during Keycloak's first start and the port opens + well before the realm exists. +- Optional HorizontalPodAutoscaler. `replicas` is omitted from the Deployment + when it is enabled, so the two do not fight on every reconcile. +- Optional PodDisruptionBudget, which fails the render if enabled alongside a + single replica. +- `helm test` check that `/health` responds. +- `extraVolumes` / `extraVolumeMounts` / `extraInitContainers` escape hatches. +- Per-environment example values for dev and production. The production example + connects as the database owner rather than the superuser, pulls secrets from + ESO, sets a PodDisruptionBudget and pod anti-affinity, and enables the pod and + container security contexts. + +### Changed from the compose stack +- Probes use `/health` (as compose does) rather than Sunbird's + `/api/docs/swagger.json`, which would also require swagger to be enabled. + A `startupProbe` covers first-start schema migration. +- Resource limits added; Sunbird's chart sets requests only. +- The three secret values are read from named Secret keys instead of `.env` + variables, and the chart renders none of them. diff --git a/charts/registry/Chart.yaml b/charts/registry/Chart.yaml new file mode 100644 index 0000000..1b9ab4c --- /dev/null +++ b/charts/registry/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: registry +description: >- + The OAN participant registry, on Sunbird RC core. Holds participant records + and the key material inbound signed requests are verified against. Requires + PostgreSQL and Keycloak. +type: application +version: 0.2.0 +appVersion: "v2.0.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - registry + - sunbird-rc + - participant +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/Sunbird-RC/sunbird-rc-core +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/registry/README.md b/charts/registry/README.md new file mode 100644 index 0000000..9c38797 --- /dev/null +++ b/charts/registry/README.md @@ -0,0 +1,244 @@ +# registry + +The OAN participant registry, on [Sunbird RC +core](https://github.com/Sunbird-RC/sunbird-rc-core). It holds participant +records and the key material that inbound signed requests are verified against. + +Configuration is ported from the verified `registry/docker-compose.yml` stack — +all 32 environment variables, at parity. This deployment is **participant records +and lookup only**: every optional Sunbird RC subsystem (credentials, DIDs, +certificates, encryption, file storage, notifications, webhooks, async) is off, +exactly as in compose. + +## What it renders + +| Resource | Notes | +|---|---| +| Deployment | Probes on `/health`, resources mandatory, init containers wait for dependencies | +| Service | `ClusterIP` on 8081 | +| ConfigMap (env) | The non-secret flags from `envConfig` | +| ConfigMap (schemas) | Entity definitions, mounted read-only | +| ServiceAccount | | +| HorizontalPodAutoscaler | Optional, off by default | +| PodDisruptionBudget | Optional, off by default | +| Ingress | Optional, off by default | +| Test Pod | `helm test` check against `/health` | + +## Dependencies + +Two things must already be running: + +1. **PostgreSQL** → `database.host`. See [`postgresql-cnpg`](../postgresql-cnpg), + which creates the database and the owner role this chart connects as. +2. **Keycloak** → `keycloak.url`. See + [`keycloak`](../keycloak). + +## Image + +`image.repository` is **empty by default, on purpose.** The image OAN will +actually deploy is not decided yet — upstream `ghcr.io`, a mirror in our own +registry, or an OAN-built image — and a default here would read as a decision +that has not been made. + +The render **fails** while it is empty: + +``` +registry: image.repository is required - set image.registry/repository/tag for this environment +``` + +That is deliberate. An empty repository would otherwise render +`image: ghcr.io/:v2.0.0`, which Helm and the API server both accept, and which +only fails later as an `ImagePullBackOff`. + +| Values file | Image | +|---|---| +| `examples/registry.dev.yaml` | `ghcr.io/sunbird-rc/sunbird-rc-core:v2.0.0` — what the compose stack runs, so dev deploys what was verified locally | +| `examples/registry.prod.yaml` | Empty, with a TODO. Not installable until filled in | + +When you do fill in production, prefer `image.digest` over `image.tag`: a tag can +be repointed at different bits, a digest cannot, which is what makes a rollback +land on the same image it did before. + +## Waiting for dependencies + +Kubernetes has no equivalent of compose's `depends_on: condition: +service_healthy`, so the chart adds init containers: + +```yaml +waitFor: + enabled: true + database: true # TCP check, host/port from `database` + keycloak: true # HTTP check on /realms/ +``` + +Both are derived from settings you already set, so there is nothing to keep in +sync. + +The Keycloak check is deliberately **HTTP against the realm endpoint**, not TCP +against the port. Keycloak imports the realm during its first start, so the port +opens well before the realm exists — a TCP check would pass too early and the +registry would come up against a Keycloak that cannot yet issue it a usable +token. + +They give up after `waitFor.timeoutSeconds` (default 300) so a genuinely missing +dependency shows as a failed init container in `kubectl describe`, rather than an +endless wait. + +## Verifying an install + +```bash +helm test registry -n oan-registry +``` + +Runs a Pod that checks `/health`. It deliberately does not test an authenticated +endpoint: that needs a real token, and a failure there would be ambiguous between +"the registry is broken" and "the Keycloak wiring is wrong". + +## Scaling + +The registry is stateless, so `autoscaling.enabled` gives it an HPA. Two things +to know: + +- Every replica opens its own connection pool, so scaling out multiplies + connections against PostgreSQL. Check the cluster's `max_connections` before + raising `maxReplicas` much. +- When autoscaling is on, the Deployment omits `replicas` entirely, so + `replicaCount` is ignored. Otherwise Helm and the HPA fight on every reconcile. + +`podDisruptionBudget.enabled` keeps a replica serving through node drains. It +**fails the render** if enabled with `replicaCount: 1`, because a PDB of +`minAvailable: 1` over a single pod blocks drains completely — set +`allowSingleReplica: true` if you want that anyway. + +## Per-environment values + +| File | For | +|---|---| +| [`examples/registry.dev.yaml`](./examples/registry.dev.yaml) | Dev: superuser connection, hand-made Secrets, no ingress | +| [`examples/registry.prod.yaml`](./examples/registry.prod.yaml) | Production: owner connection, 2 replicas, PDB, anti-affinity, security contexts on | + +Everything that differs between environments lives in these files, not in +templates. + +## Install order + +The full stack, in the only order that works: + +```bash +# 1. Database cluster. Creates BOTH databases - `registry` via bootstrap.initdb, +# `keycloak` via a CNPG Database object - each owned by its own role. +helm install registry-db charts/postgresql-cnpg \ + -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml + +# 2. Keycloak — imports the sunbird-rc realm on first start +helm install keycloak charts/keycloak \ + -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml + +# 3. MANUAL: regenerate the admin-api client secret. +# The realm export ships it masked ("**********"), so it does not work as-is. +# kubectl -n oan-registry port-forward svc/keycloak 8080:8080 +# http://localhost:8080/auth/admin -> realm sunbird-rc -> Clients +# -> admin-api -> Credentials -> Regenerate Secret +# kubectl -n oan-registry create secret generic registry-keycloak \ +# --from-literal=keycloakAdminClientSecret='' \ +# --from-literal=registryDefaultUserPassword='' + +# 4. Registry +helm install registry charts/registry \ + -n oan-registry -f charts/registry/examples/registry.dev.yaml +``` + +Step 3 is unavoidable while the realm is imported from a masked export — see +[`keycloak`](../keycloak#the-two-phase-first-install). + +## Entity schemas + +The registry treats every `.json` under its schema directory as an entity it +serves APIs for. The chart ships `files/schemas/Participant.json` and mounts the +rendered ConfigMap read-only at +`/home/sunbirdrc/config/public/_schemas`. A `checksum/schemas` annotation rolls +the pod when a schema changes. + +| You want | Set | +|---|---| +| The shipped schemas | nothing — this is the default | +| Schemas managed outside the chart | `schemas.existingConfigMap: ` | +| Extra or overriding schemas | `schemas.inline: {Name.json: ''}` | + +The render fails if no schema matches, since a registry with no entity +definitions serves nothing. + +> **The participant schema is not settled.** Per engineering-tracker #33, it was +> built and verified against the local stack but has no design issue of its own +> (#43 was closed as a duplicate of #69, which covers a different schema). +> Expect it to change, and a schema change is a chart release. + +## Authentication + +`authenticationEnabled: true` — role checks on `Participant` only apply while +this is true. Turning it off makes every endpoint unauthenticated; it is a +local-debugging switch, not a deployment option. + +Three settings must agree with Keycloak, and getting any of them wrong produces +401/403 responses that read like a permissions bug rather than a configuration +one: + +| Setting | Must be | +|---|---| +| `keycloak.url` | Exactly the issuer Keycloak puts in the `iss` claim, including `/auth` | +| `keycloak.realm` | A realm that is actually imported (`sunbird-rc`) | +| `keycloak.adminClientSecret` | The **regenerated** `admin-api` secret, never the masked one from the export | + +`OAUTH2_RESOURCES_0_URI` is derived as `/realms/`, so it +cannot drift from `keycloak.url`. + +## Database + +Defaults match compose: the registry connects to database `registry` as the +`postgres` superuser, sharing that database with Keycloak. + +**CNPG disables superuser access by default**, so reproducing the compose +arrangement requires `enableSuperuserAccess: true` on the database chart — which +the example values set, and which makes CNPG generate +`Secret/-superuser` for the password. + +Once you are past reproducing compose, the better arrangement is: leave +superuser access off, connect the registry as the cluster's bootstrap owner +(`database.user: registry`, password from `Secret/-app`), and give +Keycloak its own database and role. Then no workload holds superuser rights. + +## Secrets + +Three values, none of which this chart ever renders: + +| Value | Setting | +|---|---| +| Database password | `database.passwordSecret` | +| `admin-api` client secret | `keycloak.adminClientSecret` | +| Default password for Keycloak users the registry creates | `defaultUserPasswordSecret` | + +This chart renders none of them — it only references Secrets, which are created +by whatever manages secrets in that environment. For dev that is a +`kubectl create secret`; the example values show the exact command. + +## Probes + +`/health`, matching the compose healthcheck. Sunbird's own chart probes +`/api/docs/swagger.json`, which additionally requires swagger to be enabled; +`/health` is the narrower check. The registry migrates its schema on first start, +so `startupProbe` carries the slow path. + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema and +[`examples/registry.dev.yaml`](./examples/registry.dev.yaml) for a +per-environment file. + +The `*_enabled: "false"` flags in `envConfig` are deliberate. Each one that is +turned on pulls in another Sunbird RC service that is not deployed — the registry +will start, then fail at the first request that needs it. + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/registry/ci/lint-values.yaml b/charts/registry/ci/lint-values.yaml new file mode 100644 index 0000000..dbdaf7b --- /dev/null +++ b/charts/registry/ci/lint-values.yaml @@ -0,0 +1,18 @@ +# Minimum values that let `helm template` run in CI. Dummy references only - +# see examples/ for real per-environment configuration. +database: + host: registry-db-rw + passwordSecret: + name: registry-db-app +keycloak: + url: http://keycloak:8080/auth + adminClientSecret: + name: registry-keycloak +defaultUserPasswordSecret: + name: registry-keycloak + +# image.repository is intentionally empty in the chart, so CI supplies one. +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-core + tag: "v2.0.0" diff --git a/charts/registry/examples/registry.dev.yaml b/charts/registry/examples/registry.dev.yaml new file mode 100644 index 0000000..94fa2aa --- /dev/null +++ b/charts/registry/examples/registry.dev.yaml @@ -0,0 +1,84 @@ +# Example: the OAN participant registry, dev environment. +# +# helm install registry charts/registry \ +# -n oan-registry -f charts/registry/examples/registry.dev.yaml +# +# Install this LAST - it needs the database, and it needs the regenerated +# admin-api client secret from Keycloak. See README.md for the full order. + +# No fullnameOverride needed: the chart is named "registry", so a release named +# "registry" already produces Service/registry. + +# The image the compose stack runs, so dev deploys what was verified locally. +# The chart default is empty on purpose - see values.yaml. +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-core + tag: "v2.0.0" + +# Connects as the OWNER of its own database, not as the superuser. Compose uses +# `postgres` because everything shares one database there; in the cluster the +# registry owns `registry` and Keycloak owns `keycloak`, so neither needs +# superuser rights. +database: + host: registry-db-rw + port: 5432 + name: registry + user: registry + # The Secret the cluster's bootstrap owner was created from. + passwordSecret: + name: registry-db-app + key: password + +keycloak: + # Must match the keycloak release's in-cluster DNS, including /auth, and must + # equal the issuer Keycloak puts in its tokens. + url: http://keycloak.oan-registry.svc.cluster.local:8080/auth + realm: sunbird-rc + adminClientId: admin-api + clientId: registry-frontend + # The REGENERATED admin-api secret. Until ESO is available: + # kubectl -n oan-registry create secret generic registry-keycloak \ + # --from-literal=keycloakAdminClientSecret='' \ + # --from-literal=registryDefaultUserPassword='' + adminClientSecret: + name: registry-keycloak + key: keycloakAdminClientSecret + +authenticationEnabled: true + +keycloakUserSetPassword: true +defaultUserPasswordSecret: + name: registry-keycloak + key: registryDefaultUserPassword + +# Ships files/schemas/*.json (currently just Participant.json) +schemas: + existingConfigMap: "" + +ingress: + enabled: false + +resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +commonLabels: + oan.in/environment: dev + +# Blocks startup until the database is reachable AND Keycloak's realm endpoint +# answers. The Keycloak check is HTTP against the realm, not TCP against the +# port: the realm is imported during Keycloak's first start, so the port opens +# well before the realm the registry needs exists. +waitFor: + enabled: true + database: true + keycloak: true + +# `helm test registry -n oan-registry` checks /health. +tests: + enabled: true diff --git a/charts/registry/examples/registry.prod.yaml b/charts/registry/examples/registry.prod.yaml new file mode 100644 index 0000000..1a56e3b --- /dev/null +++ b/charts/registry/examples/registry.prod.yaml @@ -0,0 +1,144 @@ +# Example: the OAN participant registry, production. +# +# helm install registry charts/registry \ +# -n oan-registry -f charts/registry/examples/registry.prod.yaml +# +# The differences from dev are all here, per issue #30's requirement that +# environment differences live only in values files. + +# --------------------------------------------------------------------------- +# TODO: set the production image. Left empty on purpose - the image OAN will +# actually deploy is not decided yet (upstream ghcr, a mirror in our own +# registry, or an OAN-built image), and guessing here would be a guess that +# looks like a decision. +# +# The chart FAILS TO RENDER while repository is empty, which is deliberate: an +# empty repository would otherwise produce "ghcr.io/:v2.0.0" and only fail later +# as an ImagePullBackOff. So this example is not installable until it is filled +# in. +# +# When filling it in, prefer a digest over a tag. A tag can be repointed at +# different bits; a digest cannot, which is what makes a rollback land on the +# same image it did before. +# +# image: +# registry: +# repository: +# tag: "v2.0.0" +# digest: "sha256:..." +# --------------------------------------------------------------------------- +image: + registry: "" + repository: "" + tag: "" + digest: "" + +# Two replicas minimum: one can be drained without an outage, and the PDB below +# needs somewhere to move traffic. +replicaCount: 2 + +database: + host: registry-db-rw + port: 5432 + name: registry + # Production connects as the database OWNER, not the superuser. This is the + # main security difference from dev: the registry has no rights beyond its own + # database. Requires the cluster's bootstrap owner to be `registry`. + user: registry + passwordSecret: + name: registry-db-app + key: password + +keycloak: + url: http://keycloak.oan-registry.svc.cluster.local:8080/auth + realm: sunbird-rc + adminClientId: admin-api + clientId: registry-frontend + adminClientSecret: + name: registry-keycloak + key: keycloakAdminClientSecret + +authenticationEnabled: true + +keycloakUserSetPassword: true +defaultUserPasswordSecret: + name: registry-keycloak + key: registryDefaultUserPassword + +# Secrets are created out of band; this chart renders none of them. The Secret +# named above must carry both keys: +# kubectl -n oan-registry create secret generic registry-keycloak \ +# --from-literal=keycloakAdminClientSecret='' \ +# --from-literal=registryDefaultUserPassword='' + +resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi + +# Survives a single node drain. +podDisruptionBudget: + enabled: true + minAvailable: 1 + +# Off until there is load data to size it from. Note every replica opens its own +# connection pool, so check the cluster's max_connections before enabling. +autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 4 + targetCPUUtilizationPercentage: 70 + +# Spread replicas across nodes, so one node failing does not take both. +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: registry + app.kubernetes.io/instance: registry + +# Hardened. Verify the image runs as non-root before enabling in a new +# environment - a mismatch shows up as a permission error at startup. +podSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + enabled: true + allowPrivilegeEscalation: false + runAsNonRoot: true + # Left false: the JVM writes to /tmp. Enabling it needs an emptyDir mounted + # there via extraVolumes, and verification that nothing else writes to disk. + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + +ingress: + enabled: false + # When the AWS Load Balancer Controller is installed: + # className: alb + # annotations: + # alb.ingress.kubernetes.io/scheme: internal + # alb.ingress.kubernetes.io/target-type: ip + # hosts: + # - host: registry.oan.example + # paths: + # - path: / + # pathType: Prefix + +envConfig: + logging.level.root: WARN + +commonLabels: + oan.in/environment: prod diff --git a/charts/registry/files/schemas/Participant.json b/charts/registry/files/schemas/Participant.json new file mode 100644 index 0000000..748beb7 --- /dev/null +++ b/charts/registry/files/schemas/Participant.json @@ -0,0 +1,170 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "Participant": { + "$ref": "#/definitions/Participant" + } + }, + "required": [ + "Participant" + ], + "title": "Participant", + "definitions": { + "Participant": { + "$id": "#/properties/Participant", + "type": "object", + "title": "OAN network participant", + "comment": "Flat by design: nested objects would each become their own graph node with their own osid. Every field here is a scalar or an array of scalars, so one participant is one record with one osid.", + "required": [ + "participant_id", + "display_name", + "roles", + "status", + "record_version", + "updated_at", + "signing_public_key", + "endpoint_url", + "domain" + ], + "additionalProperties": false, + "properties": { + "participant_id": { + "type": "string", + "title": "Business identifier of the participant, unique across the network" + }, + "display_name": { + "type": "string", + "title": "Human-readable name of the participant" + }, + "roles": { + "type": "array", + "title": "Roles this participant holds on the network", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "network_operator", + "provider", + "consumer" + ] + } + }, + "status": { + "type": "string", + "title": "Lifecycle status of the participant", + "enum": [ + "active", + "inactive" + ] + }, + "domain": { + "type": "string", + "title": "Kind of participant, e.g. weather, market, credit, advisory" + }, + "record_version": { + "type": "integer", + "title": "Version counter of this record, starts at 1", + "minimum": 1 + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "ISO-8601 timestamp of the last update" + }, + "endpoint_url": { + "type": "string", + "title": "Base URL where requests to this participant are sent once resolved" + }, + "endpoint_type": { + "type": "string", + "title": "What this endpoint serves", + "enum": [ + "onix", + "callback", + "api" + ] + }, + "signing_public_key": { + "type": "string", + "title": "Base64-encoded public key a receiver uses to verify this participant's signed requests" + }, + "signing_algorithm": { + "type": "string", + "title": "Algorithm of the signing key pair", + "enum": [ + "ed25519" + ] + }, + "key_valid_from": { + "type": "string", + "format": "date-time", + "title": "ISO-8601 timestamp from which the signing key is usable" + }, + "key_valid_until": { + "type": "string", + "format": "date-time", + "title": "ISO-8601 timestamp after which the signing key must not be trusted" + }, + "osid": { + "type": "string", + "title": "System id, assigned by the registry - accepted on update, never set by a client" + }, + "osOwner": { + "type": "array", + "items": { + "type": "string" + }, + "title": "System field: keycloak subject(s) that own this record" + }, + "osCreatedAt": { + "type": "string", + "title": "System field: creation timestamp" + }, + "osUpdatedAt": { + "type": "string", + "title": "System field: last update timestamp" + }, + "osCreatedBy": { + "type": "string", + "title": "System field: creator" + }, + "osUpdatedBy": { + "type": "string", + "title": "System field: last updater" + }, + "_status": { + "type": [ + "boolean", + "string", + "null" + ], + "title": "System field: storage-level soft-delete flag" + } + } + } + }, + "_osConfig": { + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ], + "uniqueIndexFields": [ + "participant_id" + ], + "indexFields": [ + "status", + "domain" + ], + "roles": [ + "admin", + "network_operator" + ], + "inviteRoles": [ + "admin", + "network_operator" + ] + } +} \ No newline at end of file diff --git a/charts/registry/templates/NOTES.txt b/charts/registry/templates/NOTES.txt new file mode 100644 index 0000000..6d6eee5 --- /dev/null +++ b/charts/registry/templates/NOTES.txt @@ -0,0 +1,35 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "registry.fullname" . }} ({{ .Values.replicaCount }} replica(s)) + - Service/{{ include "registry.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "registry.envConfigMapName" . }} +{{- if include "registry.renderSchemasConfigMap" . }} + - ConfigMap/{{ include "registry.schemasConfigMapName" . }} (entity schemas) +{{- end }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "registry.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "registry.fullname" . }} +{{- end }} + +Wired to: + database: {{ include "registry.jdbcUrl" . }} (as {{ .Values.database.user }}) + keycloak: {{ .Values.keycloak.url }} (realm {{ .Values.keycloak.realm }}) + auth: {{ if .Values.authenticationEnabled }}enabled - role checks apply{{ else }}DISABLED - no role checks{{ end }} + +In-cluster endpoint: + http://{{ include "registry.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +Check it: + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "registry.fullname" . }} + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "registry.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl http://localhost:{{ .Values.service.port }}/health + +If authenticated calls come back 401 or 403, check these three in order - they +are the usual causes, and none of them look like an auth problem in the logs: + 1. keycloak.url must match the issuer Keycloak puts in its tokens, exactly. + 2. keycloak.adminClientSecret must be the REGENERATED admin-api secret; the + one in the realm export is masked and never works. + 3. The realm ({{ .Values.keycloak.realm }}) must actually be imported. diff --git a/charts/registry/templates/_helpers.tpl b/charts/registry/templates/_helpers.tpl new file mode 100644 index 0000000..d2df577 --- /dev/null +++ b/charts/registry/templates/_helpers.tpl @@ -0,0 +1,181 @@ +{{/* +# ============================================================================ +# REGISTRY (SUNBIRD RC) CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the database, +# Keycloak and schema wiring the registry needs. +# ============================================================================ +*/}} + +{{- define "registry.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "registry.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "registry.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "registry.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "registry.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "registry.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "registry.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +JDBC URL for the registry database. +*/}} +{{- define "registry.jdbcUrl" -}} +{{- $db := .Values.database -}} +{{- printf "jdbc:postgresql://%s:%v/%s" $db.host $db.port $db.name -}} +{{- end }} + +{{/* +Name of the ConfigMap holding the entity schemas. An existing ConfigMap wins, +so schemas can be managed outside this chart. +*/}} +{{- define "registry.schemasConfigMapName" -}} +{{- if .Values.schemas.existingConfigMap -}} +{{- .Values.schemas.existingConfigMap -}} +{{- else -}} +{{- printf "%s-schemas" (include "registry.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Emits "true" when this chart renders the schemas ConfigMap itself. +*/}} +{{- define "registry.renderSchemasConfigMap" -}} +{{- if not .Values.schemas.existingConfigMap -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +The schema files shipped with the chart, as ConfigMap data entries. The registry +reads every .json under its schema directory as an entity definition, so an +empty set means a registry that knows about no entities at all. +*/}} +{{- define "registry.schemasData" -}} +{{- $files := .Files.Glob .Values.schemas.filesGlob -}} +{{- if and (not $files) (not .Values.schemas.inline) -}} +{{- fail (printf "%s: no schema files matched %q in the chart and schemas.inline is empty. The registry needs at least one entity definition." .Chart.Name .Values.schemas.filesGlob) -}} +{{- end -}} +{{- range $path, $_ := $files }} +{{ base $path }}: |- + {{- $.Files.Get $path | nindent 2 }} +{{- end }} +{{- range $name, $content := .Values.schemas.inline }} +{{ $name }}: |- + {{- $content | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Environment the registry needs beyond the flags in envConfig: everything derived +from the database and Keycloak settings, plus the three secret values. + +Mirrors registry/docker-compose.yml. `sunbird_sso_url` and OAUTH2_RESOURCES_0_URI +must agree with the issuer Keycloak actually puts in its tokens, or every request +is rejected as unauthorised. +*/}} +{{- define "registry.env" -}} +{{- $db := .Values.database -}} +{{- $kc := .Values.keycloak -}} +{{- if not $db.host }} +{{- fail (printf "%s: database.host is required - point it at the PostgreSQL primary service, e.g. registry-db-rw" .Chart.Name) }} +{{- end }} +{{- if not $db.passwordSecret.name }} +{{- fail (printf "%s: database.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) }} +{{- end }} +{{- if not $kc.url }} +{{- fail (printf "%s: keycloak.url is required, including the /auth context path, e.g. http://keycloak:8080/auth" .Chart.Name) }} +{{- end }} +{{- if not $kc.adminClientSecret.name }} +{{- fail (printf "%s: keycloak.adminClientSecret.name is required. Regenerate the admin-api client secret in the Keycloak console after the realm import, then store it." .Chart.Name) }} +{{- end }} +{{- if and .Values.keycloakUserSetPassword (not .Values.defaultUserPasswordSecret.name) }} +{{- fail (printf "%s: keycloakUserSetPassword is true, so defaultUserPasswordSecret.name is required - it is the password the registry sets on Keycloak users it creates" .Chart.Name) }} +{{- end }} +{{- $kcUrl := $kc.url | trimSuffix "/" }} +- name: connectionInfo_uri + value: {{ include "registry.jdbcUrl" . | quote }} +- name: connectionInfo_username + value: {{ $db.user | quote }} +- name: connectionInfo_password + valueFrom: + secretKeyRef: + name: {{ $db.passwordSecret.name }} + key: {{ $db.passwordSecret.key }} +- name: authentication_enabled + value: {{ .Values.authenticationEnabled | quote }} +- name: sunbird_sso_realm + value: {{ $kc.realm | quote }} +- name: sunbird_sso_url + value: {{ $kcUrl | quote }} +- name: OAUTH2_RESOURCES_0_URI + value: {{ printf "%s/realms/%s" $kcUrl $kc.realm | quote }} +- name: OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH + value: {{ $kc.rolesPath | quote }} +- name: identity_provider + value: {{ $kc.identityProvider | quote }} +- name: sunbird_sso_admin_client_id + value: {{ $kc.adminClientId | quote }} +- name: sunbird_sso_client_id + value: {{ $kc.clientId | quote }} +- name: sunbird_sso_admin_client_secret + valueFrom: + secretKeyRef: + name: {{ $kc.adminClientSecret.name }} + key: {{ $kc.adminClientSecret.key }} +- name: sunbird_keycloak_user_set_password + value: {{ .Values.keycloakUserSetPassword | quote }} +{{- with .Values.defaultUserPasswordSecret.name }} +- name: sunbird_keycloak_user_password + valueFrom: + secretKeyRef: + name: {{ . }} + key: {{ $.Values.defaultUserPasswordSecret.key }} +{{- end }} +{{- with (include "oan-common.env" . | trim) }} +{{ . }} +{{- end }} +{{- end }} + +{{/* +Dependency waits, derived from this chart's own settings so there is nothing to +keep in sync. + +The Keycloak check is HTTP against the realm endpoint, not TCP against the port: +the realm is imported during Keycloak's first start, so the port opens well +before the realm the registry needs actually exists. A TCP check would pass too +early and the registry would start against a Keycloak that cannot yet issue it a +usable token. +*/}} +{{- define "registry.waitFor" -}} +{{- $tcp := list -}} +{{- $http := list -}} +{{- if .Values.waitFor.database }} +{{- $tcp = append $tcp (dict "name" "database" "host" .Values.database.host "port" .Values.database.port) -}} +{{- end -}} +{{- if .Values.waitFor.keycloak }} +{{- $url := printf "%s/realms/%s" (.Values.keycloak.url | trimSuffix "/") .Values.keycloak.realm -}} +{{- $http = append $http (dict "name" "keycloak" "url" $url) -}} +{{- end -}} +{{- $tcp = concat $tcp (.Values.waitFor.extraTcp | default list) -}} +{{- $http = concat $http (.Values.waitFor.extraHttp | default list) -}} +{{- include "oan-common.waitFor" (dict "ctx" . "tcp" $tcp "http" $http) -}} +{{- end }} diff --git a/charts/registry/templates/configmap-schemas.yaml b/charts/registry/templates/configmap-schemas.yaml new file mode 100644 index 0000000..d2ed8c8 --- /dev/null +++ b/charts/registry/templates/configmap-schemas.yaml @@ -0,0 +1,18 @@ +{{- if include "registry.renderSchemasConfigMap" . }} +{{/* +Entity definitions, mounted at schemas.mountPath. The registry treats every +.json here as an entity it serves APIs for. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "registry.schemasConfigMapName" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "registry.schemasData" . | trim | nindent 2 }} +{{- end }} diff --git a/charts/registry/templates/configmap.yaml b/charts/registry/templates/configmap.yaml new file mode 100644 index 0000000..aa8ade7 --- /dev/null +++ b/charts/registry/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "registry.envConfigMapName" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with (include "oan-common.envConfigMapData" . | trim) }} +data: + {{- . | nindent 2 }} +{{- else }} +data: {} +{{- end }} diff --git a/charts/registry/templates/deployment.yaml b/charts/registry/templates/deployment.yaml new file mode 100644 index 0000000..c03f6f5 --- /dev/null +++ b/charts/registry/templates/deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "registry.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "registry.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + checksum/schemas: {{ include "registry.schemasData" . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "registry.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + {{- $init := include "registry.waitFor" . | trim }} + {{- $extraInit := "" }} + {{- with .Values.extraInitContainers }} + {{- $extraInit = toYaml . | trim }} + {{- end }} + {{- if or $init $extraInit }} + initContainers: + {{- with $init }} + {{- . | nindent 8 }} + {{- end }} + {{- with $extraInit }} + {{- . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "registry.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- include "registry.env" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "registry.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + - name: schemas + mountPath: {{ .Values.schemas.mountPath }} + readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: schemas + configMap: + name: {{ include "registry.schemasConfigMapName" . }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/registry/templates/hpa.yaml b/charts/registry/templates/hpa.yaml new file mode 100644 index 0000000..4157d8d --- /dev/null +++ b/charts/registry/templates/hpa.yaml @@ -0,0 +1,51 @@ +{{- if .Values.autoscaling.enabled }} +{{/* +The registry is stateless - it holds no session state and writes everything to +PostgreSQL - so it scales horizontally. Two things to keep in mind: + + - Every replica opens its own connection pool, so scaling out multiplies + connections against the database. Check `max_connections` on the cluster + before raising maxReplicas much. + - Autoscaling takes over `replicas`, so replicaCount is ignored once this is + enabled. The Deployment deliberately omits `replicas` in that case, or the + two would fight on every reconcile. +*/}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "registry.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- with .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/registry/templates/ingress.yaml b/charts/registry/templates/ingress.yaml new file mode 100644 index 0000000..285444a --- /dev/null +++ b/charts/registry/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "registry.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/registry/templates/poddisruptionbudget.yaml b/charts/registry/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..d9ae912 --- /dev/null +++ b/charts/registry/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Caps how many pods can be voluntarily disrupted at once - node drains, cluster +upgrades, autoscaler scale-down. + +With replicaCount: 1 a PDB of minAvailable: 1 blocks drains entirely, because +evicting the only pod would breach it. That is why this is disabled by default +and should be enabled together with replicaCount >= 2. +*/}} +{{- if and (eq (int .Values.replicaCount) 1) (not .Values.podDisruptionBudget.allowSingleReplica) }} +{{- fail (printf "%s: podDisruptionBudget is enabled with replicaCount 1, which blocks node drains entirely. Raise replicaCount to 2+, or set podDisruptionBudget.allowSingleReplica=true if you accept that." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "registry.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/registry/templates/service.yaml b/charts/registry/templates/service.yaml new file mode 100644 index 0000000..13ccd91 --- /dev/null +++ b/charts/registry/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "registry.selectorLabels" . | nindent 4 }} diff --git a/charts/registry/templates/serviceaccount.yaml b/charts/registry/templates/serviceaccount.yaml new file mode 100644 index 0000000..b80401c --- /dev/null +++ b/charts/registry/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "registry.serviceAccountName" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/registry/templates/tests/test-connection.yaml b/charts/registry/templates/tests/test-connection.yaml new file mode 100644 index 0000000..b6ad6e0 --- /dev/null +++ b/charts/registry/templates/tests/test-connection.yaml @@ -0,0 +1,36 @@ +{{- if .Values.tests.enabled }} +{{/* +`helm test` check: the registry answers on /health. + +Deliberately does not test an authenticated endpoint: that needs a real token, +and a failure there would be ambiguous between "registry is broken" and +"Keycloak wiring is wrong". Health tells you the pod is serving; the NOTES +explain how to check the auth chain. +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "registry.fullname" . }}-test-connection + labels: + {{- include "registry.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: test + image: {{ printf "%s/%s:%s" .Values.tests.image.registry .Values.tests.image.repository .Values.tests.image.tag | quote }} + command: + - /bin/sh + - -c + - | + set -e + url="http://{{ include "registry.fullname" . }}:{{ .Values.service.port }}/health" + echo "GET $url" + wget -q -O- --timeout=10 "$url" + echo "" + echo "OK: registry is serving /health" + resources: + {{- toYaml .Values.tests.resources | nindent 8 }} +{{- end }} diff --git a/charts/registry/values.yaml b/charts/registry/values.yaml new file mode 100644 index 0000000..f85b7cd --- /dev/null +++ b/charts/registry/values.yaml @@ -0,0 +1,353 @@ +# ============================================================================ +# registry - default values +# +# The OAN participant registry, on Sunbird RC core. Defaults mirror the verified +# compose stack (registry/docker-compose.yml): participant records and lookup +# only, with every optional Sunbird RC subsystem (credentials, DIDs, +# certificates, file storage, notifications, webhooks) switched off. +# +# Depends on two things already running: +# - PostgreSQL -> database.host +# - Keycloak -> keycloak.url +# +# Minimum you must set: +# - database.host, database.passwordSecret.name +# - keycloak.url, keycloak.adminClientSecret.name +# - defaultUserPasswordSecret.name +# ============================================================================ + +replicaCount: 1 + +# --------------------------------------------------------------------------- +# Image +# +# repository is intentionally EMPTY: the image OAN deploys is not settled yet, and +# a default here would look like a decision that has not been made. The render +# fails while it is empty rather than producing "ghcr.io/:v2.0.0", which would +# only surface later as an ImagePullBackOff. +# +# The compose stack runs ghcr.io/sunbird-rc/sunbird-rc-core:v2.0.0, which is what +# the dev example values use. Set this per environment. +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: "" + # RELEASE_VERSION in the compose stack + tag: "v2.0.0" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: true + +service: + type: ClusterIP + port: 8081 + targetPort: 8081 + annotations: {} + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: registry.local + paths: + - path: / + pathType: Prefix + tls: [] + +# --------------------------------------------------------------------------- +# Database +# +# Compose runs the registry as the postgres superuser against the same database +# Keycloak uses. On a cluster, prefer a dedicated owner role: set user to the +# CNPG cluster's bootstrap owner and give Keycloak its own database. +# --------------------------------------------------------------------------- +database: + # PostgreSQL primary service, e.g. "registry-db-rw" + host: "" + port: 5432 + name: registry + user: postgres + # Secret holding the database password. Required. + passwordSecret: + name: "" + key: password + +# --------------------------------------------------------------------------- +# Keycloak +# +# `url` must include the /auth context path the legacy Keycloak serves under, +# and must be the URL Keycloak itself puts in the `iss` claim. If they disagree, +# every authenticated request is rejected - and the failure looks like a +# permissions problem, not a configuration one. +# +# The keycloak chart prints the exact value to use in its NOTES. +# --------------------------------------------------------------------------- +keycloak: + # e.g. http://keycloak.oan-registry.svc.cluster.local:8080/auth + url: "" + realm: sunbird-rc + # Confidential client the registry uses for admin operations on Keycloak + adminClientId: admin-api + # Client tokens are issued to + clientId: registry-frontend + rolesPath: realm_access.roles + identityProvider: dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl + # Secret holding the admin-api client secret. + # + # The realm export ships this masked, so it is NOT usable until someone + # regenerates it: Keycloak console -> realm sunbird-rc -> Clients -> admin-api + # -> Credentials -> Regenerate Secret. First install is two-phase because of + # this. + adminClientSecret: + name: "" + key: keycloakAdminClientSecret + +# Role checks on Participant only apply while this is true. +authenticationEnabled: true + +# Whether the registry sets a password on Keycloak users it creates. +keycloakUserSetPassword: true +# The password it sets. Required while keycloakUserSetPassword is true. +defaultUserPasswordSecret: + name: "" + key: registryDefaultUserPassword + +# --------------------------------------------------------------------------- +# Entity schemas +# +# Every .json under the registry's schema directory is an entity definition. +# The chart ships the participant schema and mounts it at mountPath. +# +# NOTE: this schema was built and verified against the local stack, but per +# engineering-tracker #33 it has no design issue of its own yet. Expect it to +# change - a schema change is a chart release. +# --------------------------------------------------------------------------- +schemas: + filesGlob: "files/schemas/*.json" + mountPath: /home/sunbirdrc/config/public/_schemas + # Manage schemas outside this chart instead of shipping them + existingConfigMap: "" + # Extra or overriding schemas, as filename -> JSON string + inline: {} + +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +# --------------------------------------------------------------------------- +# Probes +# +# /health is what the compose healthcheck uses. Sunbird's own Helm chart probes +# /api/docs/swagger.json instead, which additionally requires swagger to be +# enabled; /health is the narrower, more honest check. +# +# The registry runs schema migrations on first start, so startupProbe carries +# the slow path. +# --------------------------------------------------------------------------- +startupProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 30 + +livenessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 0 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +readinessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# --------------------------------------------------------------------------- +# Dependency waiting +# +# Kubernetes has no equivalent of compose's `depends_on: condition: +# service_healthy`. Without these init containers the pod starts before its +# dependencies are up, fails, and crashloops with backoff - which recovers on +# its own but makes a first install look broken. +# --------------------------------------------------------------------------- +waitFor: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + timeoutSeconds: 300 + intervalSeconds: 3 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Wait for the database. Host and port come from `database` above. + database: true + # Wait for Keycloak's REALM endpoint, derived from keycloak.url and + # keycloak.realm - not just its port. The realm is imported during Keycloak's + # first start, so the port opens well before the realm exists, and a TCP check + # would let the registry start against a Keycloak that cannot yet issue it a + # usable token. + keycloak: true + extraTcp: [] + extraHttp: [] + +# Extra init containers, appended after the waitFor ones. Rendered verbatim. +extraInitContainers: [] + +# Extra volumes and mounts, rendered verbatim. For certificates, extra config, +# or a writable scratch directory when readOnlyRootFilesystem is enabled. +extraVolumes: [] +extraVolumeMounts: [] + +# --------------------------------------------------------------------------- +# PodDisruptionBudget +# +# Caps voluntary disruptions - node drains, cluster upgrades, autoscaler +# scale-down. Disabled by default because with a single replica a PDB of +# minAvailable: 1 blocks drains entirely; the render fails if you enable it +# anyway without acknowledging that. +# --------------------------------------------------------------------------- +podDisruptionBudget: + enabled: false + minAvailable: 1 + # Set one or the other, not both. + maxUnavailable: "" + # Enable a PDB with a single replica anyway, accepting that node drains block. + allowSingleReplica: false + +# --------------------------------------------------------------------------- +# `helm test` checks +# +# helm test -n +# +# Runs after install to confirm the service actually answers, which is the +# difference between "the pod is running" and "the deployment works". +# --------------------------------------------------------------------------- +tests: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + +# --------------------------------------------------------------------------- +# Autoscaling +# +# The registry is stateless, so it scales horizontally. Note that every replica +# opens its own connection pool, so check the database's max_connections before +# raising maxReplicas much. When enabled, replicaCount is ignored. +# --------------------------------------------------------------------------- +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + # Set to a number to also scale on memory. + targetMemoryUtilizationPercentage: "" + # Scale-up/scale-down tuning, rendered verbatim into spec.behavior. + behavior: {} + + +# --------------------------------------------------------------------------- +# ENVIRONMENT CONFIGURATION +# +# Rendered into a ConfigMap and injected with envFrom. These are exactly the +# non-secret, non-derived variables from the compose stack. Database and +# Keycloak variables are derived from the sections above instead, so they are +# not repeated here. +# +# The *_enabled flags are all false on purpose: this deployment is participant +# records and lookup only. Turning any of them on pulls in another Sunbird RC +# service that is not deployed - the registry will start and then fail at the +# first request that needs it. +# +# NEVER put secrets here. +# --------------------------------------------------------------------------- +envConfig: + search_providerName: dev.sunbirdrc.registry.service.NativeSearchService + manager_type: DefinitionsManager + expand_reference: "false" + encryption_enabled: "false" + event_enabled: "false" + idgen_enabled: "false" + claims_enabled: "false" + did_enabled: "false" + signature_enabled: "false" + certificate_enabled: "false" + filestorage_enabled: "false" + notification_enabled: "false" + notification_async_enabled: "false" + async_enabled: "false" + webhook_enabled: "false" + registry_base_apis_enable: "false" + swagger_title: OAN Registry + logging.level.root: INFO + +# Extra env vars from specific Secret keys, beyond the three wired above. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} From 2460c06ca1e172bb6ec2cdf5e44f9a7a9b742c83 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:35:27 +0530 Subject: [PATCH 17/81] docs: list the seven charts and their install order in README [#53] Records the chart inventory and the order releases have to go out in, since the database cluster has to exist before the migration Job and the services that connect to it. --- README.md | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 81364f6..ed05c3c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,146 @@ # helmcharts -Helm charts for deploying and managing platform services + +Helm charts for deploying and managing OpenAgriNet (OAN) platform services. + +## Charts + +| Chart | Type | Purpose | +|---|---|---| +| [`oan-common`](charts/oan-common) | library | Shared template helpers — names, labels, image refs, probes, resources, service account, env config, dependency waits. Renders nothing; never installed directly. | +| [`oan-template`](charts/oan-template) | application | Complete, working starter chart built on `oan-common`. Copy it to bootstrap a service chart. | +| [`postgresql-cnpg`](charts/postgresql-cnpg) | application | CloudNativePG-managed PostgreSQL cluster. One release per database. Requires the CNPG operator. | +| [`postgresql-migration`](charts/postgresql-migration) | application | Flyway migrations as a Job. Creates the per-service databases and applies versioned SQL. | +| [`keycloak`](charts/keycloak) | application | Auth for the registry, on the Sunbird RC Keycloak image. Imports the realm the registry expects. | +| [`registry`](charts/registry) | application | The OAN participant registry, on Sunbird RC core. Needs `postgresql-cnpg` and `keycloak`. | +| [`discovery`](charts/discovery) | application | The OAN Beckn discover-and-publish service. Needs `postgresql-cnpg` **with pgvector**. | + +## How they fit together + +``` +charts/ +├── oan-common/ # library chart — shared helpers +├── oan-template/ # starter chart — copy this to build a service chart +├── postgresql-cnpg/ # data store +├── postgresql-migration/# schema migrations (Flyway Job) +├── keycloak/ # auth for the registry +├── registry/ # the participant registry +└── discovery/ # the Beckn discover-and-publish service +``` + +Every chart depends on `oan-common` via `file://../oan-common`. + +## The registry stack + +Three charts, deployed in this order — the ordering is not optional: + +```bash +# 1. Database cluster. Creates BOTH databases: `registry` via bootstrap.initdb +# and `keycloak` via a CNPG Database object, each owned by its own role. +helm install registry-db charts/postgresql-cnpg -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml +# 2. Keycloak — imports the sunbird-rc realm it ships with, on first start +helm install keycloak charts/keycloak -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml +# MANUAL STEP: regenerate the admin-api client secret in the Keycloak console — +# the realm export ships it masked, so the registry cannot authenticate without this. +# 3. Registry +helm install registry charts/registry -n oan-registry -f charts/registry/examples/registry.dev.yaml +``` + +`postgresql-migration` is deliberately not in that list: both databases come from +the cluster chart, and Sunbird RC and Keycloak each manage their own schema, so +there is nothing for Flyway to apply yet. It joins the flow when OAN adds schemas +of its own. + +Their configuration is ported from the verified `registry/docker-compose.yml` +stack at exact environment-variable parity (32 for the registry, 10 for +Keycloak), so a cluster deploy reproduces what was tested locally. Each chart's +README documents where it deliberately deviates and why. Full walkthrough: +[`charts/registry/README.md`](charts/registry/README.md). + +## The discovery service + +Two charts, and one prerequisite that is easy to miss: + +```bash +# 1. Its own database — pgvector on PostgreSQL 16, with the `vector` extension +# created at bootstrap. Both are required: no stock CNPG operand image has +# pgvector, and `vector` is not a trusted extension, so the owner the service +# connects as cannot create it itself. +helm install discovery-db charts/postgresql-cnpg -n oan-discovery -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml +# 2. The service +helm install discovery charts/discovery -n oan-discovery -f charts/discovery/examples/discovery.dev.yaml +``` + +It shares no database and no Keycloak with the registry stack, so the two are +independent installs. Full walkthrough, including how the DSN and the Beckn +specification are supplied: +[`charts/discovery/README.md`](charts/discovery/README.md). + +Service charts depend on `oan-common` and call its helpers through thin +chart-local wrappers. That keeps naming, labelling, probe, resource, and secret +conventions identical across every OAN chart, and means a convention change is +one edit in the library rather than one edit per chart. + +## Quick start + +Build a service chart from the template: + +```bash +# 1. Copy the starter chart +cp -r charts/oan-template charts/oan-my-service + +# 2. In charts/oan-my-service/Chart.yaml set name: oan-my-service +# and appVersion to the image tag you deploy by default. +# Keep the oan-common dependency. + +# 3. Rename the chart-local helpers to your service name. Change only the left +# side of each define in templates/_helpers.tpl (oan-template.* -> +# oan-my-service.*); the oan-common.* include inside the body stays. This +# renames the defines and the include calls together: +grep -rl 'oan-template\.' charts/oan-my-service | xargs sed -i '' 's/oan-template\./oan-my-service./g' +# (sed -i '' is the macOS form; on Linux use sed -i) + +# 4. Set image, ports, probe paths, resources and envConfig in +# charts/oan-my-service/values.yaml + +# 5. Validate +./scripts/lint-charts.sh +helm template oan-my-service charts/oan-my-service +``` + +See [`charts/oan-common/README.md`](charts/oan-common/README.md) for the full +helper reference and +[`charts/oan-template/README.md`](charts/oan-template/README.md) for the +step-by-step adaptation guide. + +## Validation + +```bash +./scripts/lint-charts.sh +``` + +Runs `helm lint --strict` on every chart and `helm template` on every +application chart. CI runs the identical script +([`.github/workflows/helm-lint.yml`](.github/workflows/helm-lint.yml)) on pull +requests and on pushes to `main` and `development`. + +Because charts depend on `oan-common` through `file://../oan-common` and the +packaged dependency is not committed, an edit to `oan-common` only reaches a +consuming chart after `helm dependency update charts/` — or a run of the +lint script, which does it for you. + +## Conventions + +Chart naming, `version`/`appVersion` rules, changelog requirements, what every +service chart must declare, and how secrets are handled are documented in +[`CONVENTIONS.md`](CONVENTIONS.md). + +Two of those rules are enforced at render time rather than at review time: a +chart with empty `resources` fails to render, and so does an enabled probe with +no handler or with more than one. + +## Secrets + +No secret value belongs in this repository, and **no chart here renders a +Secret**. Charts reference Secrets by name; creating them is deliberately left +outside the charts, so that decision is made once rather than per chart. See +[`CONVENTIONS.md`](CONVENTIONS.md#secrets). From 92766f6cf42a16dd8eb5c84ae52bc4f472f5c8be Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 15:46:55 +0530 Subject: [PATCH 18/81] feat: serve the mapping from this repo, and stop forbidding keys on an upstream [OpenAgriNet/network-adapter#4] MAPPING_URL pointed at a personal fork. It now points at this repo's own copy of the file, the one already sitting in config/mappings/, served over the raw CDN -- so the file a reviewer reads and the file the adapter fetches are one file and cannot drift. Verified fetchable anonymously, and it is in fact the current mapping: the fork still serves the older one that echoed bapId and bppId back in the response context. The URL carries a branch, which is noted in both .env.example and the README: repoint it at the default branch on merge, or pin a tag, so a deployment is not following a moving file. The Participant schema no longer forbids role and keys on a type "upstream". Nothing reads them there -- a signature is verified against the node identity that signed it, never against an upstream -- but forbidding them refused a record a deployment might legitimately want to keep, and the rule read as though an upstream were prevented from ever signing. The conditional is deleted rather than emptied: an if/then with an empty then reads like a rule and enforces nothing. The node conditional is untouched and still requires role and keys, which is the half that matters. Verified against a stack on the published images: an upstream with role and keys is now accepted, one without still is, and a node without them is still refused. The full Postman flow passes 20 assertions with the adapter fetching the mapping from this repo -- confirmed in its log alongside the signature it verified, exp.oan.dev's, resolved from the registry. --- docker-deployment/.env.example | 27 +++++++----- docker-deployment/README.md | 41 +++++++++++-------- .../config/registry/schemas/Participant.json | 33 +-------------- 3 files changed, 43 insertions(+), 58 deletions(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index 5c241e3..2f6d09c 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -112,14 +112,19 @@ PROVIDER_CAPABILITY=openagrinet:WeatherObservation # The mapping the provider adapter fetches, request and response in one file. # The registry row holds the full URL and the adapter fetches it verbatim. # -# This defaults to the published copy, which is the same file found in -# config/mappings/ -- that copy is there to read and to fork, not to be served -# from here. A mapping has to be published somewhere the adapter can reach -# before it can be tested, so what this stack exercises is what consumers -# actually fetch. Serving the local copy would prove the file works and prove -# nothing about the file anyone else reads. -# -# To change it: fork it, publish the copy (a raw GitHub URL is fine) and put -# that URL in the ProviderSchema row. This variable is only the default the -# README example uses. -MAPPING_URL=https://raw.githubusercontent.com/ameersohel45/oan-mappings/main/mausamgram/weather-observation.select.yaml +# This repo's own copy, served over the raw CDN -- the same file that sits in +# config/mappings/ beside these configs, so what the adapter fetches and what +# a reader reviews are one file and cannot drift. +# +# It is a URL and not a path because the registry publishes the full URL and +# the adapter fetches it verbatim. A mapping therefore has to be reachable +# before it can be tested, which means what this stack exercises is exactly +# what any consumer fetches. +# +# Note the branch in the path. Once this merges, change it to the default +# branch, or pin a tag so a deployment is not following a moving file. +# +# To change the mapping: edit config/mappings/, push, and the next cache +# expiry picks it up -- or publish a fork anywhere that serves raw text over +# https and put that URL in the ProviderSchema row instead. +MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml diff --git a/docker-deployment/README.md b/docker-deployment/README.md index 12b9043..f63c42c 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -134,8 +134,11 @@ published as. Keycloak builds the token's issuer from these headers, and the registry validates that issuer against the internal address. Get it wrong and the registry rejects the token with a 401 and an empty body. -**Row one — the API itself.** Type `upstream`: it has no role and no keys, -because it has never heard of Beckn. +**Row one — the API itself.** Type `upstream`: an ordinary HTTP API this +deployment calls. It does not sign anything and nothing verifies it, so no +keys are needed — the signing in this flow is between adapters, on the three +`node` identities `bin/setup.py` seeded. `role` and `keys` are accepted on an +upstream if a deployment wants to record them; nothing reads them. ```sh curl -s -X POST http://127.0.0.1:8081/api/v1/Participant \ @@ -163,7 +166,7 @@ curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema \ "action": "select", "method": "GET", "path": "/get-daily", - "mappings": "https://raw.githubusercontent.com/ameersohel45/oan-mappings/main/mausamgram/weather-observation.select.yaml", + "mappings": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml", "timeoutMs": 15000, "retryMax": 2, "status": "active" @@ -176,10 +179,12 @@ Things worth knowing about these two calls: - **No `{"Participant": {...}}` wrapper.** The registry takes the record itself. A wrapper comes back as `extraneous key [Participant] is not permitted`. -- **An `upstream` carries no `role`, no `keys` and no credential.** It has - never heard of Beckn, and nothing held in the registry is ever sent to it — - the adapter presents credentials from its own config, naming environment - variables. The schema refuses `role` or `keys` on an upstream. +- **An `upstream` needs no `role` and no `keys`**, and no credential is held + for it here. It has never heard of Beckn, and nothing in the registry is + sent to it — the adapter presents credentials from its own config, naming + environment variables. `role` and `keys` are permitted if a deployment wants + to record them, but nothing reads them: a signature is verified against the + `node` identity that signed it. - **The three roles are `consumer`, `provider` and `network`**, and they apply to `node` rows only — the three `setup.py` creates. A node also needs at least one key, published as bare base64 with no encoding label in front of @@ -346,17 +351,21 @@ config/ ## About the mapping file -`config/mappings/` holds the same file the published mappings repository -serves. It is there to **read and to fork** — not to be served from here. +`config/mappings/` holds the mapping this deployment uses, and `MAPPING_URL` +points at **this repo's own copy** over GitHub's raw CDN. So the file a reader +reviews and the file the adapter fetches are one file, and cannot drift. -The registry row holds a full URL and the adapter fetches it verbatim, so a -mapping has to be published somewhere the adapter can reach before it can be -tested. What this stack exercises is therefore what consumers actually fetch. -Serving the local copy would prove the file works and prove nothing about the -file anyone else reads. +It is a URL rather than a path because the registry publishes the full URL and +the adapter fetches it verbatim — which means a mapping has to be reachable +before it can be tested, and what this stack exercises is exactly what any +consumer fetches. -To change the mapping: fork it, publish the copy anywhere that serves raw text -over https, and put that URL in the `mappings` field of the ProviderSchema row. +Note the branch in that URL. Once this merges, point it at the default branch, +or pin a tag so a deployment is not following a moving file. + +To change the mapping: edit the file here and push, or publish a fork anywhere +that serves raw text over https and put that URL in the `mappings` field of +the ProviderSchema row. The adapter caches a mapping for `cacheTTL` (one minute, in the adapter config) and GitHub's raw CDN caches for about five, so give an edit a few minutes to diff --git a/docker-deployment/config/registry/schemas/Participant.json b/docker-deployment/config/registry/schemas/Participant.json index a809ec1..39c93e1 100644 --- a/docker-deployment/config/registry/schemas/Participant.json +++ b/docker-deployment/config/registry/schemas/Participant.json @@ -46,7 +46,7 @@ "pattern": "^https?://(?!.*\\.\\.)[A-Za-z0-9][A-Za-z0-9.:-]*(/[A-Za-z0-9._~%-]+)*$" }, "role": { - "description": "What this party does on the network. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs.", + "description": "What this party does on the network. consumer asks. provider answers. network exposes publish and discover, answering discover from published catalogs. Required on a node. Permitted on an upstream, where nothing reads it.", "type": "string", "enum": [ "consumer", @@ -55,7 +55,7 @@ ] }, "keys": { - "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header.", + "description": "Plural for rotation: an overlap where old and new are both valid. The sender names which it used by osid in the Authorization header. Required on a node, since a node's signatures are verified against them. Permitted on an upstream, which does not sign anything this stack verifies.", "type": "array", "minItems": 1, "maxItems": 8, @@ -93,35 +93,6 @@ } } } - }, - { - "description": "An upstream does not speak Beckn: it has no role on the network and no keys we verify.", - "if": { - "properties": { - "type": { - "const": "upstream" - } - }, - "required": [ - "type" - ] - }, - "then": { - "not": { - "anyOf": [ - { - "required": [ - "role" - ] - }, - { - "required": [ - "keys" - ] - } - ] - } - } } ] }, From e4f67c471be2bc2fe5837cf706e3e7c633e7846b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Wed, 2 Sep 2026 16:02:54 +0530 Subject: [PATCH 19/81] fix: make the on_select answer validate against the Beckn v2 spec [OpenAgriNet/network-adapter#4] The mapping this deployment serves produced an answer the pinned LTS spec refuses, which matters more here than anywhere: MAPPING_URL points at this copy, so this is the file the adapter fetches. status.descriptor.code was QUOTED; the spec's enum is DRAFT, ACTIVE and CLOSED. DRAFT is also the honest value -- a quote is a draft commitment, since nothing is committed until init and confirm. each resource lacked quantity, which Commitment.resources requires while the spec defines no quantity property and no Quantity schema. The defect is upstream; the consequence was ours, since an answer without it fails validation for any consumer who validates. Carried over verbatim from the adapter's reference copy, where the same fix is committed with its test. --- .../mausamgram/weather-observation.select.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml index e63b014..0b1fe69 100644 --- a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml +++ b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml @@ -188,8 +188,12 @@ response: | "contract": { "commitments": [ { + /* DRAFT, not QUOTED. The Beckn v2 status enum is DRAFT, ACTIVE + and CLOSED, and a quote is still a draft: nothing is committed + until init and confirm. QUOTED read better and validated + nowhere -- base schema validation refuses it. */ "status": { - "descriptor": { "code": "QUOTED", "name": "Quoted" } + "descriptor": { "code": "DRAFT", "name": "Draft" } }, /* The offer is echoed, but its references are not: the request named the abstract point forecast, and the answer returns the @@ -214,6 +218,12 @@ response: | "resources": [$map($days, function($day) { { "id": $resourceId($day), + /* Required by Commitment.resources in the spec, which defines + no quantity property and no Quantity schema anywhere -- a + defect upstream. One resource is one day's observation, so + one. Omitting it makes every answer fail validation for a + consumer who validates. */ + "quantity": 1, "resourceAttributes": { "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", From b003f3bfa2c4e8604c46494914d9ffe1b0321ceb Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Thu, 3 Sep 2026 14:36:19 +0530 Subject: [PATCH 20/81] feat: ship the flow as a Postman collection [OpenAgriNet/network-adapter#4] Anyone cloning this branch got the stack but not a way to exercise it beyond copying curl out of the README. The collection under postman-collection/ runs the whole thing: a write token, the provider's two registry rows, both registry searches, publish, discover and select. Every value is prefilled in the collection's own variables rather than in a separate environment file, so the file is useful to whoever receives it with no second import. That includes providerId and capability, set to the values .env.example uses -- which matters, because those two form the binding key the provider adapter answers to, and a row naming anything else is refused 404 with the response naming the module rather than the cause. Typing the id by hand is how that happens. One variable is left to set: upstreamBaseUrl, which the deployment cannot know. The requests carry assertions, so a run reports whether the stack is healthy rather than merely returning 200s: that the three adapter identities exist with signing keys, that the provider is registered as an upstream with no role or keys, that publish is ACCEPTED, that select answers a resource per forecast day whose status is in the spec's enum and whose every resource carries a quantity, and that no party is named in the answer. Every payload in it validates against the pinned Beckn v2 LTS spec and, for resourceAttributes, against openagrinet:WeatherObservation v0.1. It carries no credential: the token variable ships empty and is filled by request 1. --- docker-deployment/README.md | 11 +- .../OAN-dev-flow.postman_collection.json | 455 ++++++++++++++++++ .../postman-collection/README.md | 32 ++ 3 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json create mode 100644 docker-deployment/postman-collection/README.md diff --git a/docker-deployment/README.md b/docker-deployment/README.md index f63c42c..b90bedd 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -109,7 +109,14 @@ Three participants, one per adapter. That is what `setup.py` seeded. ## Register the provider -Two rows. Both by hand, and both need a token. +**Quickest path: import `postman-collection/`.** It does everything in this +section and the end-to-end test after it — a token, the provider's two rows, +both registry searches, publish, discover and select — with every value +prefilled to match this deployment. Set one variable, `upstreamBaseUrl`, to a +URL the VM can reach for your API, and run the requests in order. + +The rest of this section is the same thing as curl, if you would rather see it +step by step. Two rows, both by hand, and both need a token. Get the upstream API's URL first. If it is tunnelled from a laptop: @@ -347,6 +354,8 @@ config/ instance.yaml.example optional override; see the compose file mappings/ mausamgram/ the request and response transformation +postman-collection/ the whole flow as a Postman collection, with the + deployment's own values prefilled ``` ## About the mapping file diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json new file mode 100644 index 0000000..1aabf3b --- /dev/null +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -0,0 +1,455 @@ +{ + "info": { + "name": "OAN dev \u2014 registry to select", + "description": "The whole flow against a docker-deployment stack: register a provider, look at what the registry holds, publish a catalogue, discover it, and select from it.\n\nSET ONE VARIABLE AND RUN. Everything is prefilled in the collection's own variables, including providerId and capability as DEPLOYED -- so the binding key this creates already matches the adapter that will serve it. Only upstreamBaseUrl needs filling in: a URL the VM can reach for your API, which the deployment cannot know.\n\nWhy that matters. The provider adapter answers exactly one binding key, participantId|capabilityCode, rendered into its config from the stack's .env. A ProviderSchema row naming anything else is refused 404 'this module serves no capability matching the request'. providerId and capability here are prefilled to the values that deployment uses, so they agree by default -- change them only if the deployment changed.\n\nPREREQUISITE. bin/setup.py must have run on the stack. It seeds the three ADAPTER identities and renders their configs; this collection adds the provider, because the provider's base URL is not the stack's to know.\n\nRun in order the first time -- request 1 issues the token requests 2 and 3 need. Re-running is safe: the registry is append-only, so a repeat create reports the existing row rather than changing anything.\n\nPorts bind to loopback on the VM, so from a workstation tunnel first:\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 you@the-vm", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "keycloakUrl", + "value": "http://127.0.0.1:8080" + }, + { + "key": "registryUrl", + "value": "http://127.0.0.1:8081/api/v1" + }, + { + "key": "expAdapterUrl", + "value": "http://127.0.0.1:9202" + }, + { + "key": "providerAdapterUrl", + "value": "http://127.0.0.1:9200" + }, + { + "key": "keycloakRealm", + "value": "sunbird-rc" + }, + { + "key": "keycloakClientId", + "value": "registry-frontend" + }, + { + "key": "registryUser", + "value": "no-user" + }, + { + "key": "registryPassword", + "value": "no-user-password" + }, + { + "key": "providerId", + "value": "my-weather-api", + "description": "AS DEPLOYED: PROVIDER_PARTICIPANT_ID in the stack's .env. With this and capability below it forms the binding key the provider adapter answers to. Change only if that deployment was changed." + }, + { + "key": "capability", + "value": "openagrinet:WeatherObservation", + "description": "AS DEPLOYED: PROVIDER_CAPABILITY in the stack's .env." + }, + { + "key": "upstreamBaseUrl", + "value": "https://YOUR-TUNNEL-SUBDOMAIN.ngrok-free.app", + "description": "THE ONE VALUE TO SET. Your upstream API, reachable from the VM. The deployment cannot know it." + }, + { + "key": "upstreamPath", + "value": "/get-daily", + "description": "The path on your API for the select action. One leading slash, no empty segment." + }, + { + "key": "mappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml", + "description": "Full URL of the request/response mapping, fetched verbatim by the adapter. This is the deployment's own copy in the helmcharts repo, so what the adapter fetches is the reviewed file. Note the branch in the path -- repoint it once that merges." + }, + { + "key": "networkId", + "value": "oan-dev" + }, + { + "key": "domain", + "value": "oan-dev" + }, + { + "key": "catalogId", + "value": "cat-weather-point-forecast" + }, + { + "key": "token", + "value": "", + "description": "Filled in by request 1." + } + ], + "item": [ + { + "name": "1. Registry \u2014 get a write token", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Forwarded-Host", + "value": "keycloak:8080" + }, + { + "key": "X-Forwarded-Proto", + "value": "http" + } + ], + "url": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", + "description": "Keycloak issues the bearer token the two registry writes below need.\n\nThe two X-Forwarded-* headers are not optional. Keycloak runs behind PROXY_ADDRESS_FORWARDING and builds the token's issuer from them, and the registry validates that issuer against keycloak:8080 -- the CONTAINER-INTERNAL address, not whatever KEYCLOAK_PORT is published as. Get it wrong and every write is refused 401 with an empty body.\n\nThe token is saved to the {{token}} collection variable.", + "body": { + "mode": "urlencoded", + "urlencoded": [ + { + "key": "client_id", + "value": "{{keycloakClientId}}" + }, + { + "key": "grant_type", + "value": "password" + }, + { + "key": "username", + "value": "{{registryUser}}" + }, + { + "key": "password", + "value": "{{registryPassword}}" + } + ] + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const j = pm.response.json();", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", + "pm.collectionVariables.set(\"token\", j.access_token);" + ] + } + } + ] + }, + { + "name": "2. Registry \u2014 create the provider", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "description": "The upstream API itself, and it deliberately carries NO KEYS.\n\nWHERE ARE THE SIGNING KEYS, THEN. The registry holds two kinds of participant, and only one of them signs:\n\n type \"node\" a Beckn participant. REQUIRES role and keys. These are the three adapters -- exp, network and provider -- and bin/setup.py created them, generating a keypair for each, publishing the public half here and rendering the private half into that adapter's own config. They are what signAck signs with and what validateSign verifies against.\n\n type \"upstream\" an ordinary HTTP API our provider adapter calls. It has never heard of Beckn: it does not sign its responses and nothing verifies them. role and keys are permitted on it but not expected, and nothing in the adapter reads them: a signature is verified against the node identity that signed it, never against an upstream.\n\nSo the signing in this flow is entirely between adapters, on the identities setup.py seeded. Run request 4 to see all four rows side by side: three nodes with keys, this one without.\n\nCredentials for calling this API, if it needs any, are not held here either -- the provider adapter's own config names the environment variables they come from, so nothing secret is in the registry.\n\nSet upstreamBaseUrl to a URL the VM can reach; an ngrok https URL if the API runs on a laptop.\n\nNo {\"Participant\": {...}} wrapper: the registry takes the record itself, and a wrapper is refused as 'extraneous key [Participant] is not permitted'.\n\nThe registry is append-only. There is no update, delete is soft and keeps the unique index, so a participantId can never be reused -- get this wrong and pick a new id.", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"Weather API\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{upstreamBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const p = pm.response.json().params;", + "pm.test(\"the registry answered (created, or already present)\", () => pm.expect([\"SUCCESSFUL\",\"UNSUCCESSFUL\"]).to.include(p.status));", + "if (p.status !== \"SUCCESSFUL\") {\n // Expected on a re-run: the registry is append-only, so a second create of\n // the same id is a duplicate-key failure, not a problem. Anything else is.\n console.log(\"not created -- already present, or refused:\", p.errmsg);\n}" + ] + } + } + ] + }, + { + "name": "3. Registry \u2014 create the capability binding", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema", + "description": "Which capability this provider answers, and how to call it.\n\nbindingKey is {{providerId}}|{{capability}}, and both are prefilled to what the adapter was deployed with -- so this row matches by default. It is one key: while the adapter carries a single configured key this is a one-provider deployment, and onboarding another means editing .env, re-running bin/setup.py and restarting the provider adapter. A registry entry on its own is not enough.\n\nHow a mismatch shows up:\n payload names a provider the adapter is not configured for\n 404 NET_ENTITY_NOT_FOUND, 'this module serves no capability matching the request'\n adapter configured for the key but this row missing\n 502 with an empty body; the reason is only in\n docker compose logs provider-adapter | grep 'no call plan'\n\npath must start with one / and carry no empty segment; //get-daily is refused by the schema and by the adapter. mappings is the FULL url of the mapping file, fetched verbatim.\n\nNo {\"ProviderSchema\": {...}} wrapper -- the registry takes the record itself.", + "body": { + "mode": "raw", + "raw": "{\n \"bindingKey\": \"{{providerId}}|{{capability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{capability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{upstreamPath}}\",\n \"mappings\": \"{{mappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const p = pm.response.json().params;", + "pm.test(\"the registry answered (created, or already present)\", () => pm.expect([\"SUCCESSFUL\",\"UNSUCCESSFUL\"]).to.include(p.status));", + "if (p.status !== \"SUCCESSFUL\") {", + " // Expected on a re-run: the registry is append-only, so a second create of", + " // the same id is a duplicate-key failure, not a problem. Anything else is.", + " console.log(\"not created -- already present, or refused:\", p.errmsg);", + "}", + "console.log(\"binding key created:\", pm.variables.get(\"providerId\") + \"|\" + pm.variables.get(\"capability\"),", + " \"-- this must be what the provider adapter was deployed with\");" + ] + } + } + ] + }, + { + "name": "4. Registry \u2014 search participants", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{registryUrl}}/Participant/search", + "description": "What the registry now holds, and the answer to 'where are the keys'.\n\nExpect four rows: the three ADAPTER identities that bin/setup.py seeded, each type \"node\" with a role and a published signing key, and the provider created above, type \"upstream\" with neither. That split is the whole trust model -- adapters sign and are verified against these published keys; the upstream API is just an HTTP endpoint behind one of them.\n\nNote the key shape on a node: bare base64, no encoding label, identified by the osid the registry assigned on write, and no keyId or use field.", + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().data;", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const mine = rows.filter(r => r.participantId === pm.variables.get(\"providerId\"));", + "pm.test(\"the provider is there\", () => pm.expect(mine.length).to.eql(1));", + "pm.test(\"it is an upstream, with no role and no keys\", () => {", + " pm.expect(mine[0].type).to.eql(\"upstream\");", + " pm.expect(mine[0].role).to.be.undefined;", + " pm.expect(mine[0].keys).to.be.undefined;", + "});", + "", + "// The other side of the trust model: the adapters DO publish keys, and those", + "// are what every signature in this flow is verified against. If these are", + "// missing, bin/setup.py has not run and nothing will authenticate.", + "const nodes = rows.filter(r => r.type === \"node\");", + "pm.test(\"the adapter identities are seeded, each with a signing key\", () => {", + " pm.expect(nodes.length).to.be.at.least(3);", + " nodes.forEach(n => {", + " pm.expect(n.role, n.participantId + \" has no role\").to.be.a(\"string\");", + " pm.expect((n.keys || []).length, n.participantId + \" publishes no key\").to.be.above(0);", + " });", + "});" + ] + } + } + ] + }, + { + "name": "5. Registry \u2014 search provider bindings", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{registryUrl}}/ProviderSchema/search", + "description": "The capability bindings. One row per provider per capability, each carrying the call plan the provider adapter resolves against at request time -- which is why repointing a provider at a new URL is a registry write and not a config change.", + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().data;", + "const want = pm.variables.get(\"providerId\") + \"|\" + pm.variables.get(\"capability\");", + "pm.test(\"the binding is there\", () => pm.expect(rows.map(r => r.bindingKey)).to.include(want));" + ] + } + } + ] + }, + { + "name": "6. Publish \u2014 one catalogue, one resource", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{providerAdapterUrl}}/publish", + "description": "context.action is catalog/publish, which is the name the Beckn v2 spec gives this action at /catalog/publish -- a bare \"publish\" is not a spec action and would be refused by schema validation. The callback comes back as catalog/on_publish.\n\nEnters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service.\n\nThe caller signs nothing, and the body names no party: identity travels in the Authorization header's keyId, taken from the adapter's own keyManager config.\n\n/publish sits at the root, outside /oan/ -- it is not part of this adapter's Beckn surface, so it must not shadow it.\n\nThe catalogue carries no offers. Nothing requires them: a catalogue of resources alone is legal, and the discovery service stores the offers member only when one is sent. select does not read them either -- it carries its own offer in the request, and the answer echoes that one back with its resource references rewritten.", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-02T06:00:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"{{catalogId}}\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"WX-01\",\n \"name\": \"Point weather forecast catalogue\",\n \"shortDesc\": \"Daily point weather forecast\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, several days ahead.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"WX-01\",\n \"name\": \"Weather API\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Weather forecast for a single point\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const r = pm.response.json().message.results[0];", + "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", + "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(pm.variables.get(\"catalogId\")));" + ] + } + } + ] + }, + { + "name": "7. Discover \u2014 find it", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{expAdapterUrl}}/oan/discover", + "description": "Experience adapter -> network layer -> discovery service. The experience adapter is the only one that takes an unsigned request, because the app in front of it is inside the trust boundary -- which is what makes this callable with no signature.", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"domain\": \"{{domain}}\",\n \"transactionId\": \"3a7d9c11-2b4e-4f60-8a1c-5d6e7f809a1b\",\n \"messageId\": \"4b8e0d22-3c5f-4071-9b2d-6e7f8091a2b3\",\n \"timestamp\": \"2026-09-02T06:20:00.000Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const m = pm.response.json().message;", + "pm.test(\"on_discover\", () => pm.expect(pm.response.json().context.action).to.eql(\"on_discover\"));", + "pm.test(\"at least one catalogue\", () => pm.expect((m.catalogs || []).length).to.be.above(0));" + ] + } + } + ] + }, + { + "name": "8. Select \u2014 quote it", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{expAdapterUrl}}/oan/select", + "description": "Experience adapter -> provider adapter -> the upstream API. Never touches the network layer.\n\nThe resource is a full openagrinet:WeatherObservation v0.1, in OnDemand mode -- which is what a select IS: asking a provider to obtain the information rather than carrying it. The pack requires informationMode and subjectCategories in either mode, and in OnDemand it requires supportedObservationTypes, supportedParameters and geographicGranularities while FORBIDDING parameters. So a request states what it wants observed, never the readings themselves.\n\nquantity is required on each resource by the pinned Beckn v2 spec, which defines no quantity property and has no Quantity schema -- a defect upstream. Any value satisfies it; without one, schema validation refuses the request.\n\nThe answer comes back in Direct mode with ONE RESOURCE PER FORECAST DAY, ids derived from each date, and the offer's resourceIds rewritten to match. Its context carries only correlation ids -- no party is named in either direction.", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-09-02T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:point-forecast\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:open-data\",\n \"resourceIds\": [\n \"res:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"WX-01\",\n \"name\": \"Weather API\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A 404 here is the binding-key mismatch, not a wiring problem: the response", + "// names the module, not the cause.", + "if (pm.response.code === 404) {", + " console.log(\"404: the provider adapter serves no capability matching this payload.\",", + " \"offer.provider.id and resourceAttributes.@type must equal the deployed key:\",", + " pm.variables.get(\"providerId\") + \"|\" + pm.variables.get(\"capability\"));", + "}", + "// A 502 with no body is the other half: the adapter is configured for the key", + "// but no ProviderSchema row resolves. Only the adapter log says so.", + "if (pm.response.code === 502) {", + " console.log(\"502: no call plan resolved. Run request 5 to confirm the row exists, then\",", + " \"docker compose logs provider-adapter | grep 'no call plan'\");", + "}", + "", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", + "const c = b.message.contract.commitments[0];", + "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", + "const ids = c.resources.map(r => r.id);", + "pm.test(\"the offer references only resources returned\", () => {", + " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", + "});", + "pm.test(\"no party named in the answer\", () => {", + " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", + "});" + ] + } + } + ] + } + ] +} diff --git a/docker-deployment/postman-collection/README.md b/docker-deployment/postman-collection/README.md new file mode 100644 index 0000000..673134f --- /dev/null +++ b/docker-deployment/postman-collection/README.md @@ -0,0 +1,32 @@ +# Postman collection + +`OAN-dev-flow.postman_collection.json` runs the whole flow against a stack +brought up from the compose file beside it: a write token, the provider's two +registry rows, both registry searches, publish, discover and select. + +Import it and set **one** variable — `upstreamBaseUrl`, a URL the VM can reach +for your upstream API. Everything else is prefilled in the collection itself, +including `providerId` and `capability`, which are set to the values this +deployment's `.env.example` uses. So the binding key the collection creates +already matches the adapter that will serve it. + +Two things to know: + +- **`bin/setup.py` must have run first.** It seeds the three adapter + identities and renders their configs. The collection adds only the provider, + because the provider's base URL is not the stack's to know. +- **Run the requests in order the first time.** Request 1 issues the token that + requests 2 and 3 need. Re-running is safe: the registry is append-only, so a + repeat create reports the existing row rather than changing anything. + +The requests carry assertions, so a run tells you whether the stack is +actually healthy rather than just returning 200s. Among them: that the three +adapter identities exist with signing keys, that publish is `ACCEPTED`, that +`select` answers with a resource per forecast day, that its status is in the +spec's enum, and that every resource carries a `quantity`. + +If ports are bound to loopback on the VM, tunnel first and the defaults work +unchanged: + + ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ + -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 you@the-vm From 3631fc183d4697ed77ee5a01ab069ed9cdc26d95 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:45:07 +0530 Subject: [PATCH 21/81] feat: add nginx edge and observability tiers to the docker stack [#4] Splits the stack across two networks. Nginx Proxy Manager sits only on oan-edge and is the one container publishing on a routable interface; everything else binds 127.0.0.1 and is reached over an SSH tunnel. NPM's Forward Hostname is a free-text field, so the network split is what bounds the blast radius of a wrong entry -- names on oan-internal do not resolve from there. Adds HyperDX behind an observability profile, and the version-controlled nginx that NPM includes on its own: a rate-limit zone for the unsigned experience layer, and a global deny on /publish, which the provider adapter mounts with no signature validation. The npm-custom mount is deliberately not :ro. NPM's s6 init rewrites and chowns everything under /data/nginx, so a read-only mount makes the prepare service exit 1 and nginx never starts -- while the container still reports Up, because s6 itself is alive. Also ignores .env.* so a `cp .env .env.bak` before an edit cannot be committed; the existing pattern was an exact match on .env only. --- docker-deployment/.gitignore | 14 + docker-deployment/README.md | 450 +++++++++++++++++- .../config/gateway/npm-advanced/exp.conf | 20 + .../config/gateway/npm-advanced/registry.conf | 47 ++ .../config/gateway/npm-custom/http_top.conf | 22 + .../gateway/npm-custom/server_proxy.conf | 30 ++ docker-deployment/docker-compose.yml | 345 ++++++++++++-- 7 files changed, 885 insertions(+), 43 deletions(-) create mode 100644 docker-deployment/config/gateway/npm-advanced/exp.conf create mode 100644 docker-deployment/config/gateway/npm-advanced/registry.conf create mode 100644 docker-deployment/config/gateway/npm-custom/http_top.conf create mode 100644 docker-deployment/config/gateway/npm-custom/server_proxy.conf diff --git a/docker-deployment/.gitignore b/docker-deployment/.gitignore index e50986d..d812134 100644 --- a/docker-deployment/.gitignore +++ b/docker-deployment/.gitignore @@ -12,3 +12,17 @@ config/adapters/provider.yaml # A local override for the discovery service, if you make one. config/discovery/instance.yaml + +# HyperDX's optional extra environment, if you bring one. +.env.docker + +# Any copy of .env. The pattern above is an exact match, so a `cp .env +# .env.bak` before an edit -- which is the natural thing to do -- produces a +# file holding every live credential that git will happily offer to commit. +.env.* +!.env.example + +# Nginx Proxy Manager keeps its routing table, its account and its +# certificates in the npm-data and npm-letsencrypt volumes, so there is +# nothing here to ignore -- and nothing here to review either. Back those +# volumes up; they are the only copy. diff --git a/docker-deployment/README.md b/docker-deployment/README.md index b90bedd..30589b4 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -16,6 +16,14 @@ Running here: - **discovery** — catalogue search, plus its own Postgres. - **three adapters** — experience, network and provider. Same image, three configs. +- **gateway** — Nginx Proxy Manager, the only container that publishes on a + routable interface. Routes to the three adapters, and issues and renews the + Let's Encrypt certificates from its own UI. Profile `gateway`. +- **hyperdx** — ClickStack: OTLP ingest, ClickHouse, and the UI over it. + Profile `observability`. + +The last two are behind profiles because neither is needed to exercise the +stack, and HyperDX is the heaviest thing here. Deliberately **not** here: @@ -29,30 +37,308 @@ Deliberately **not** here: ## Reaching it -Ports bind to `127.0.0.1` on the VM by default. That is deliberate. Behind them -are a Keycloak whose admin password ships as `admin`, and a registry whose -write token anyone who has read `.env.example` can mint. On a VM's public -interface that is the whole network's identity records, writable. +Two doors, and which one you use depends on what you are reaching. + +### The adapters — through Nginx Proxy Manager + +NPM owns 80 and 443 and is the whole public surface. Unlike a config file, its +routing table is **rows in a SQLite database** inside the `npm-data` volume — +so the setup below is a one-time click-through, and that volume is the only +copy of the result. Back it up. + +**First boot.** Tunnel to the admin UI (it is bound to loopback on purpose, +see below) and change the shipped login immediately: + +```sh +ssh -L 81:127.0.0.1:81 -N you@the-vm +``` + +Open `http://127.0.0.1:81`. It logs in with `admin@example.com` / `changeme`, +which is live from first boot until you change it, and it forces a change on +first use. Do that before creating anything. + +**Then one proxy host per adapter.** Hosts → Proxy Hosts → Add Proxy Host: + +| Domain | Forward Hostname | Port | Then | +|---|---|---|---| +| `exp.oan.example.com` | `exp-adapter` | 9202 | paste `config/gateway/npm-advanced/exp.conf` into **Advanced** | +| `network.oan.example.com` | `network-adapter` | 9201 | — | +| `provider.oan.example.com` | `provider-adapter` | 9200 | — | + +Scheme `http` for all three: TLS terminates at NPM, and the hop to an adapter +is inside `oan-edge`. Turn on **Block Common Exploits**; leave **Websockets +Support** off, since nothing here uses them. + +Three hosts rather than one host with path prefixes, because a rate limit or a +block then attaches to a whole hostname instead of being expressed as a regex +in a textarea — and each gets its own certificate. A request arriving with a +`Host` NPM does not know gets NPM's default page, not an adapter. + +**Certificates.** SSL tab → Request a new SSL Certificate → Force SSL → HTTP +Validation. That needs two things to be true, and both are easy to miss: + +- a public DNS **A record** per hostname, pointing at the VM's address — an + Elastic IP, unless you enjoy redoing this after every stop/start; +- **port 80 open to `0.0.0.0/0`**, not to your address. Let's Encrypt fetches + `http:///.well-known/acme-challenge/…` from its own servers, whose + addresses you do not get to enumerate. A security group scoped to your IP + makes issuance fail with a challenge timeout, which looks nothing like a + firewall problem in the NPM log. + +If 80 must stay closed, use **DNS Validation** instead: NPM ships the certbot +Route 53 plugin, so you give it an access key with `route53:ChangeResourceRecordSets` +on the zone and it never needs an inbound request. That is the better answer on +AWS anyway, and it is the only one that works for a wildcard. + +Renewal is NPM's job from then on, and it uses the same validation method — so +a DNS record or an SG rule that was only temporarily correct will fail silently +in sixty days. + +### What is *not* reachable, and why that holds + +`POST /publish` returns 403 on all three hosts. This is not optional +hardening — it is the one thing standing between the public internet and an +unauthenticated write into the catalogue. + +The provider adapter mounts two modules: `/oan/` verifies the sender's +signature against the registry, but `oanProviderPublish` is mounted at `/` with +**no signature check at all**, because its intended caller is the provider's +own catalogue system inside the trust boundary. A proxy host pointed at +`provider-adapter:9200` therefore exposes `/publish` to anyone. NPM's UI +offers no way to route a host while withholding one path, so the block lives in +`config/gateway/npm-custom/server_proxy.conf`, which NPM includes in **every** +proxy host's server block automatically — a mounted file, not a click, and so +not something to remember on one host out of three. + +To let a real catalogue system publish, give it a tunnel or put it in the VPC +and let it reach `provider-adapter:9200` directly. Do not turn that `deny` into +an `allow`: an endpoint with no credential to check does not belong on a public +edge, and "an address allowlist in front of it" is a statement about the +network, which is where it should be made. + +And the deeper reason a UI-configured proxy is safe here at all: **NPM sits on +`oan-edge` only.** "Forward Hostname" is a free-text field, so anyone with the +admin password can type `registry`, `keycloak` or `discovery-db` into it — and +on that network none of those names resolve and none of those addresses are +routable. The blast radius of a wrong click is bounded to the tier that is +public anyway. Moving NPM onto `oan-internal` to "make things easier" would +remove that bound and put the registry's write API one form field away from the +internet. + +### Which nginx config is loaded, and which is a paste job + +Worth being exact about, because the two look alike in the repo: + +| File | How it applies | +|---|---| +| `config/gateway/npm-custom/http_top.conf` | **Automatic.** NPM includes it at the top of its `http` block. Declares the `exp` rate-limit zone and `limit_req_status 429`. | +| `config/gateway/npm-custom/server_proxy.conf` | **Automatic.** Included in every proxy host's server block. Holds the `/publish` deny. | +| `config/gateway/npm-advanced/exp.conf` | **Manual.** Paste into the experience host's Advanced tab. Applies `limit_req` to that host only, since a 10 r/s ceiling on signed peer traffic would throttle for no security gain. | +| `config/gateway/npm-advanced/registry.conf` | **Manual.** Paste into the registry host's Advanced tab, if you create one. Reduces the host to the two search endpoints and 403s the rest. | + +The manual one is in a file anyway because NPM's Advanced field is a textarea +in a database row: nothing diffs it and nothing reviews it. Keeping the source +here means the rule can be read even though the running copy cannot. + +### Adding a route for another service + +Deliberately two steps, and the first one is in git rather than in the UI. + +NPM is on `oan-edge`, where only the three adapters resolve. A proxy host +pointed at `registry` or `hyperdx` does not quietly work — it 502s, because +there is no route. So publishing something new is a change to +`docker-compose.yml` that a reviewer sees, followed by a click. The UI alone +cannot widen what is public. That is the property worth keeping; everything +below is about how to spend it deliberately. + +**Step 1 — put the service on `oan-edge`.** In `docker-compose.yml`, add the +network to that service. Keep `oan-internal` if it talks to anything else in +the stack, and keep the loopback publish or drop it as you like — NPM reaches +the container port directly, not the published one: + +```yaml + some-service: + networks: [oan-internal, oan-edge] +``` + +Then `docker compose up -d some-service nginx-proxy-manager`. NPM needs the +restart to pick up a name it could not resolve before. + +**Step 2 — add the proxy host.** UI → Hosts → Proxy Hosts → Add Proxy Host. +Domain `some.oan.example.com`, scheme `http`, Forward Hostname the **compose +service name** (`some-service`, not `oan-some-service` and not an IP), Forward +Port the **container** port. Then the SSL tab as with the adapters, and a DNS +A record before you request the certificate. + +**A service that is not in this compose file** — a provider API on another +host, something in the VPC — needs no step 1 at all. NPM has egress, so put +its address or hostname straight into Forward Hostname. Nothing about the +network split is involved, and nothing about that service becomes reachable +from inside this stack. + +**A second path on an existing domain** does not need a new host either. Open +the host → Custom Locations → add e.g. `/v2` forwarding to another service. +That keeps one certificate and one DNS record, at the cost of NPM's generated +config growing a location block you cannot see in the UI's main view. + +#### Worked example: the registry + +`registry` is already on `oan-edge` in `docker-compose.yml`, so step 1 is +done — but read the comment there before you use it, because the mechanics are +the easy part. + +**What is actually reachable once you route it.** `POST /api/v1/Participant/search` +takes no token; that is the call a peer needs, and publishing it is defensible. +Everything else under `/api/v1/` is a write, and writes need a Keycloak token. +Those are unobtainable from outside **today for one reason only**: Keycloak +publishes on `127.0.0.1`. The safety of this route therefore rests on a +decision made elsewhere in the compose file. Publish Keycloak later and the +registry's write surface opens along with it, with nothing on this host +changing to say so. + +There is a second wrinkle even for someone who has a token. The registry +validates a token's issuer against `http://keycloak:8080/auth/realms/…`, the +**container-internal** address, which is why the token request further down +this README carries `X-Forwarded-Host: keycloak:8080`. A token minted through +any other hostname is rejected with a 401 and an empty body. So "it 401s +through the proxy but works over the tunnel" is expected, not a proxy bug. + +**Create the host.** Hosts → Proxy Hosts → Add: + +| Field | Value | +|---|---| +| Domain | `registry.oan.example.com` | +| Scheme | `http` | +| Forward Hostname | `registry` — the compose service name, not `oan-registry` | +| Forward Port | `8081` — the **container** port. Not `REGISTRY_PORT`, which is only what loopback publishes it as | +| Block Common Exploits | on | + +**Then both guards, before you point anything at it.** + +1. Advanced tab → paste `config/gateway/npm-advanced/registry.conf`. That + reduces the host to `Participant/search` and `ProviderSchema/search` and + 403s everything else. It is an allowlist rather than a list of things to + block, because SunbirdRC uses POST for both search and create — no method + rule separates a read from a write, so a denylist is a list someone has to + keep complete forever. + +2. Access Lists → Add, then assign it on the host's Details tab. **Satisfy Any + off**, so an address *and* a password are needed. The path filter is not + authentication: search returns the full participant list — public keys, + baseUrls, who is on this network — to anyone who reaches it. + +**Check it does what you think:** + +```sh +curl -s -o /dev/null -w '%{http_code}\n' -X POST \ + https://registry.oan.example.com/api/v1/Participant/search \ + -u user:pass -H 'Content-Type: application/json' -d '{"filters":{}}' # 200 + +curl -s -o /dev/null -w '%{http_code}\n' -X POST \ + https://registry.oan.example.com/api/v1/Participant \ + -u user:pass -H 'Content-Type: application/json' -d '{}' # 403 + +curl -s -o /dev/null -w '%{http_code}\n' \ + https://registry.oan.example.com/api/v1/Participant/search # 401 +``` + +403 on the second is the Advanced paste; 401 on the third is the Access List. +If either returns 200, one of the two guards is not attached — and the failure +is silent, so this is worth re-running after any NPM change. + +If you decide against the route, take `oan-edge` back off `registry` in +`docker-compose.yml` rather than only deleting the proxy host. An attached +service is one form field away from being public. + +#### Before you route the ones already here + +Three of the internal services will look like obvious candidates. They are +not equivalent: + +| | What routing it publishes | +|---|---| +| **discovery** | Read-mostly catalogue search. The most defensible of the three, and still: it answers unauthenticated, and `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` with nothing behind it in this build. Put an Access List on it. | +| **registry** | The network's identity records. Reads are unauthenticated, writes need a Keycloak token. Publishable, but only cut down to the search endpoints and behind an Access List — see below. | +| **keycloak** | An admin console with a realm imported from a file that ships `no-user` / `no-user-password` and an admin-api client secret. Do not publish it. | +| **hyperdx** | `clickstack-local` runs single-user with **no login at all**. Publishing it hands over every trace and log the stack has collected. If it must be shared, switch to `clickstack-all-in-one` and set up a team first. | +| **registry-db, discovery-db** | No. Use `docker compose exec`, or a tunnel. | + +The pattern: publishing a service that has no authentication of its own means +the edge is now its only authentication. NPM can be that, but only if you say +so explicitly. + +#### Putting authentication in front of one + +NPM's **Access Lists** are the built-in answer, and they are per-host: UI → +Access Lists → Add. Two independent tabs — + +- **Authorization**: username/password pairs, enforced as HTTP basic auth. +- **Access**: `allow`/`deny` rules by address or CIDR. -So reach it over a tunnel: +**Satisfy Any** decides how they combine, and the default is the one people +get wrong. *Any* means an allowed address gets in without a password — fine +for "the office network, or a password from anywhere". Turn it **off** for +"an allowed address **and** a password", which is what you want in front of +anything that has no auth of its own. + +Then assign the list on the proxy host's Details tab. It applies to the whole +host, including any Custom Locations under it. + +Basic auth is not a substitute for a real access control, and it travels in a +header on every request — so it is worth having only over HTTPS, which is the +other reason to get the certificate before the route. + +#### The cost of each addition + +Every service you attach to `oan-edge` is one form field away from being +public, because that is exactly what the network split buys and spending it is +irreversible by clicking. Keeping `oan-edge` small is what keeps "someone got +into the NPM admin UI" a bounded incident rather than an open question about +the registry. + +So: add the network in the same change that adds the proxy host, not in +advance "so it's ready". And when a route is retired, take the service back +off `oan-edge` rather than only deleting the host in NPM. + +### Everything else — through an SSH tunnel + +The registry, Keycloak, discovery, the HyperDX UI and NPM's own admin UI +publish on `127.0.0.1` only: ```sh -ssh -L 9202:127.0.0.1:9202 -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 you@the-vm +ssh -L 81:127.0.0.1:81 \ + -L 8080:127.0.0.1:8080 \ + -L 8081:127.0.0.1:8081 \ + -L 8082:127.0.0.1:8082 \ + -L 8085:127.0.0.1:8085 \ + -N you@the-vm ``` -Then everything below works against `127.0.0.1` on your own machine. +NPM admin on 81, Keycloak on 8080, the registry on 8081, discovery on 8082, +HyperDX on 8085 (adjust to your `.env`). Both Postgres instances publish no +port at all; reach them with `docker compose exec registry-db psql …`. + +The admin UI is on loopback rather than published-and-firewalled, which is what +the port comment in most NPM compose examples suggests. It ships with a known +default login and it is the one surface on this box that can mint certificates +and re-point every public route; a security group is a second system to keep in +step with that, and a loopback bind is not. -Set `BIND_ADDR=0.0.0.0` in `.env` only once something in front is terminating -TLS and authenticating, and only after the credentials in `.env` have been -changed. +There is no `BIND_ADDR` any more. It used to move every published port onto the +public interface at once, which is a footgun once something exists to expose +the one tier that should be reachable. ## Before you start On the VM: -- Docker with Compose v2, logged in to wherever the images live if it is - private — `docker login ghcr.io` +- Docker with Compose **v2.24 or newer**, logged in to wherever the images live + if it is private — `docker login ghcr.io`. The version floor is the + `env_file: required: false` on the HyperDX service, which is what lets an + absent `.env.docker` be absent instead of fatal. - Python 3 and the `cryptography` package — `pip install cryptography` +- 16 GB of RAM if you run the `observability` profile — ClickHouse alone wants + 2-4 GB on top of the two JVM services. 8 GB is workable without it. And a URL the VM can reach for the upstream provider API. If that API runs on someone's laptop, [ngrok](https://ngrok.com/) or any equivalent tunnel gives it @@ -88,6 +374,9 @@ python3 bin/setup.py # 3. now the adapters docker compose up -d + +# 4. the edge and the telemetry stack, both opt-in +docker compose --profile gateway --profile observability up -d ``` **Do not run a bare `docker compose up -d` for step 1.** An adapter config is @@ -107,6 +396,24 @@ curl -s -X POST http://127.0.0.1:8081/api/v1/Participant/search \ Three participants, one per adapter. That is what `setup.py` seeded. +And through the gateway, once the proxy hosts exist: + +```sh +# the routed surface +curl -s -o /dev/null -w '%{http_code}\n' \ + https://exp.oan.example.com/oan/search # reaches the adapter + +# the two that matter more +curl -s -o /dev/null -w '%{http_code}\n' \ + https://provider.oan.example.com/publish # 403 -- the deny is loaded +curl -s -o /dev/null -w '%{http_code}\n' \ + http://the-vm-ip/ # NPM default page, no adapter +``` + +That 403 is the check worth repeating after any NPM change: it is the only +evidence that `npm-custom/server_proxy.conf` is still mounted, and losing the +mount silently opens an unauthenticated catalogue write. + ## Register the provider **Quickest path: import `postman-collection/`.** It does everything in this @@ -257,6 +564,37 @@ The experience adapter is the only one that takes an unsigned request — the experience app is inside the trust boundary, so there is no network signature to check. That is what makes this testable with a plain curl. +## Telemetry + +`docker compose --profile observability up -d` brings up HyperDX on +`127.0.0.1:8085` (tunnel to reach it) with OTLP on 4317/4318. It is +`clickstack-local`, not `clickstack-all-in-one`: local runs single-user with no +team to create and no ingestion key to mint, which is what makes `up -d` the +whole setup step — and also why it must stay on loopback, since there is no +login in front of it. + +**What actually arrives today is less than the wiring suggests, and that is +worth knowing before you go looking for traces that are not there.** + +- **discovery** reads `OTEL_EXPORTER` and `OTEL_EXPORTER_OTLP_ENDPOINT` into + its config, and nothing in the current build consumes them — the only + OpenTelemetry packages in its `go.mod` are indirect. So `OTEL_EXPORTER` + stays `none` by default; setting it to `otlp` emits nothing rather than + failing. When the exporter is wired, `OTEL_EXPORTER=otlp` in `.env` is the + whole change and the endpoint already points here. +- **the three adapters** get `OTEL_EXPORTER_OTLP_ENDPOINT` and + `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`. Whether that image's SDK reads + them is unverified in either direction — the image is pulled and its source + is not in this repo. Nothing depends on the answer: an absent collector makes + an OTLP exporter drop spans, not fail a request. +- **container logs go nowhere near HyperDX** without something to ship them. + `docker compose logs -f ` remains the way to read them. Shipping + them would mean an OTel collector with a `filelog` receiver over + `/var/lib/docker/containers`, which is not in this stack. + +So treat this profile as the destination being ready and in one place, rather +than as observability that is switched on. + ## How a request flows Three paths, and which adapter answers is the whole design: @@ -333,10 +671,22 @@ back as a signed NACK with a `SCH_*` code and the JSON path that failed. ## The layout ``` -docker-compose.yml the whole stack +docker-compose.yml the whole stack. Read it in tiers -- the banner + comments are the structure: registry, discovery, + adapters, observability (profile), edge (profile) .env.example copy to .env bin/setup.py keys, the three adapter rows, the adapter configs config/ + gateway/ + npm-custom/ mounted to /data/nginx/custom, which NPM includes + http_top.conf on its own: the rate-limit zone declaration, + server_proxy.conf and the /publish deny that every proxy host gets + npm-advanced/ + exp.conf NOT loaded -- paste into the experience host's + Advanced tab. Kept here because a textarea in + NPM's database is not reviewable. + The routing table itself is not a file: it is + rows in the npm-data volume. adapters/ exp.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered @@ -438,6 +788,80 @@ Add this to the adapter service in the compose file: and, if the build itself is what fails, `network: host` under its `build:`. +### The gateway will not start + +```sh +docker compose logs nginx-proxy-manager +docker compose exec nginx-proxy-manager nginx -t +``` + +`unknown limit_req zone "exp"` means the `npm-custom` mount is missing, not +that the Advanced paste is wrong — the zone is declared in +`npm-custom/http_top.conf` and has to load before any server block that +references it. + +### 502 from a host that worked yesterday + +Almost always a recreated adapter. NPM writes a literal `proxy_pass` hostname, +which nginx resolves at reload and then caches; a `docker compose restart` of +an adapter keeps its address, but a recreate does not. + +```sh +docker compose restart nginx-proxy-manager +``` + +The hand-written config this replaced avoided the whole failure mode by routing +every upstream through a variable and Docker's resolver. NPM generates its own +config, so that is simply the cost of the UI. + +### The certificate request fails + +In order of likelihood: + +- **Port 80 is not open to the world.** HTTP-01 validation arrives from Let's + Encrypt's servers, not from you. An SG rule scoped to your IP fails here with + a challenge timeout that reads like a DNS problem. +- **DNS does not point here yet**, or points at an address the instance lost on + its last stop/start. Check with `dig +short `, and attach an Elastic IP + if you intend to stop the VM. +- **Rate limited.** Let's Encrypt allows 5 failed validations per hostname per + hour. Once you hit it, fix the cause and wait — retrying is what keeps you + there. Use their staging environment while debugging. +- **Renewal will fail the same way in sixty days** if the DNS record or the SG + rule was only temporarily correct, and nothing will tell you at the time. + +On AWS, DNS validation with the Route 53 plugin sidesteps the first two +entirely, and is the only option for a wildcard. + +### A request through the gateway returns NPM's default page + +The `Host` header does not match any proxy host — a missing DNS record, a +typo in the domain field, or a request made against the raw IP. NPM answers +unknown hosts itself and never consults an adapter, so this says nothing about +whether the adapter is healthy. + +### 403 on /publish + +Working as intended, on every host. See "What is *not* reachable" above; the +fix is not in NPM. + +### 429 on the experience host + +The rate limit, at 10 r/s per address with a burst of 20. A collection run that +trips it is telling you something real about the caller — but if you need +headroom for a load test, raise `rate=` in `npm-custom/http_top.conf` and +restart the gateway. + +### Locked out of the admin UI + +The account lives in the `npm-data` volume, and there is no reset flow. Recreate +the volume and you also lose every proxy host and certificate. Back it up: + +```sh +docker run --rm -v docker-deployment_npm-data:/data -v "$PWD":/backup \ + alpine tar czf /backup/npm-data.tgz -C /data . +``` + ## Starting over ```sh diff --git a/docker-deployment/config/gateway/npm-advanced/exp.conf b/docker-deployment/config/gateway/npm-advanced/exp.conf new file mode 100644 index 0000000..bfc5f59 --- /dev/null +++ b/docker-deployment/config/gateway/npm-advanced/exp.conf @@ -0,0 +1,20 @@ +# NOT loaded automatically. This is a paste job. +# +# NPM UI -> Hosts -> Proxy Hosts -> the experience-layer host -> Advanced -> +# paste this into "Custom Nginx Configuration" -> Save. +# +# It lives in a file, in this repo, because NPM's Advanced field is a textarea +# in a SQLite row: nothing diffs it, nothing reviews it, and a restore from the +# npm-data volume is the only thing that remembers it. Keeping the source here +# means the rule can be read and reviewed even though the running copy cannot. +# +# It applies to the experience host ONLY, which is the reason it is not in +# npm-custom/server_proxy.conf: /exp is app-facing and unsigned, while the +# network and provider hosts carry signed peer traffic that a 10 r/s ceiling +# would throttle for no security gain. +# +# The zone itself is declared in npm-custom/http_top.conf, which IS loaded +# automatically. If nginx rejects this paste with "unknown limit_req zone", +# that mount is missing rather than this snippet being wrong. + +limit_req zone=exp burst=20 nodelay; diff --git a/docker-deployment/config/gateway/npm-advanced/registry.conf b/docker-deployment/config/gateway/npm-advanced/registry.conf new file mode 100644 index 0000000..936f3c0 --- /dev/null +++ b/docker-deployment/config/gateway/npm-advanced/registry.conf @@ -0,0 +1,47 @@ +# NOT loaded automatically. Paste into the registry proxy host's Advanced tab +# ("Custom Nginx Configuration"), then Save. +# +# It reduces a host that would otherwise forward the whole registry API to the +# two endpoints a network peer actually needs, and 403s everything else. +# +# ---------------------------------------------------------- why an allowlist +# +# Because a denylist here fails open. SunbirdRC uses POST for both search and +# create -- POST /api/v1/Participant/search reads, POST /api/v1/Participant +# writes -- so no method rule separates them, and a list of paths to block is +# a list someone has to keep complete forever. +# +# NPM gives no clean way to say "only these paths". Its generated `location /` +# catches everything, and a second `location /` in this box is a duplicate +# that nginx refuses to start on. Exact-match `location =` blocks can only +# subtract, which is the denylist again. +# +# So the decision happens in the rewrite phase, before location matching, with +# the one `if` construction that is documented as safe: `if` containing +# nothing but `return`. Default deny, then name what is allowed. +# +# --------------------------------------------------------------- what is not +# +# This is a path filter, not authentication. It does not stop anyone who +# reaches it from reading the full participant list -- public keys, baseUrls, +# who is on this network -- because that is precisely what search returns. +# Pair it with an NPM Access List (Satisfy Any OFF) if that list is not +# something you would publish. + +set $registry_allowed 0; + +# The Beckn-facing reads. Both are POST with a filter body; the (\?|$) anchor +# keeps /searchsomething from matching the prefix. +if ($request_uri ~ "^/api/v1/(Participant|ProviderSchema)/search(\?|$)") { + set $registry_allowed 1; +} + +# Belt and braces: search is POST, and anything else arriving at that path is +# not the call this host exists to serve. +if ($request_method != POST) { + set $registry_allowed 0; +} + +if ($registry_allowed = 0) { + return 403; +} diff --git a/docker-deployment/config/gateway/npm-custom/http_top.conf b/docker-deployment/config/gateway/npm-custom/http_top.conf new file mode 100644 index 0000000..33f3651 --- /dev/null +++ b/docker-deployment/config/gateway/npm-custom/http_top.conf @@ -0,0 +1,22 @@ +# Included by NPM at the TOP of its http block, automatically, from +# /data/nginx/custom/http_top.conf. No UI step -- this file being mounted is +# the whole configuration. +# +# http_top rather than http.conf because a limit_req_zone has to be declared +# before any server block that references it, and NPM's generated proxy hosts +# are server blocks. + +# The rate-limit bucket for the experience layer. Declaring the zone costs +# nothing until a server block opts in with `limit_req` -- which is a per-host +# decision and therefore lives in that host's Advanced tab, not here. See +# config/gateway/npm-advanced/exp.conf. +# +# 10 r/s per address with a burst of 20 absorbs a Postman collection run while +# still bounding what is, at bottom, an open relay into the network: /exp/ +# takes UNSIGNED requests, so anyone who reaches it can originate one as this +# deployment's consumer. +limit_req_zone $binary_remote_addr zone=exp:10m rate=10r/s; + +# 503 says "the server is unwell"; 429 says "you are going too fast", which is +# what actually happened and the only one of the two a client can act on. +limit_req_status 429; diff --git a/docker-deployment/config/gateway/npm-custom/server_proxy.conf b/docker-deployment/config/gateway/npm-custom/server_proxy.conf new file mode 100644 index 0000000..2399014 --- /dev/null +++ b/docker-deployment/config/gateway/npm-custom/server_proxy.conf @@ -0,0 +1,30 @@ +# Included by NPM inside EVERY proxy host's server block, automatically, from +# /data/nginx/custom/server_proxy.conf. No UI step. +# +# Because it applies to every host, only rules that are correct everywhere +# belong here. There is exactly one, and it matters more than the rest of this +# stack's edge configuration combined. + +# The provider adapter mounts TWO modules. /oan/ is its Beckn surface and +# verifies the sender's signature against the registry. But oanProviderPublish +# is mounted at `/` with NO validateSign at all, because its intended caller is +# the provider's own catalogue system, inside the trust boundary -- the same +# reason the experience adapter takes unsigned calls. +# +# So a proxy host pointed at provider-adapter:9200 exposes, at /publish, +# an unauthenticated "write anything into the catalogue" endpoint. NPM's UI +# gives no way to route a host while withholding one path, and this is not a +# thing to leave to remembering an Advanced-tab paste on one host out of three. +# +# An exact-match location beats NPM's generated `location /`, so this wins +# without conflicting with it. On the exp and network hosts it denies a path +# their adapters do not serve, which costs nothing. +# +# To open it for a real catalogue system outside this VM, do NOT edit this to +# `allow`. Give that caller a tunnel, or put it in the VPC and reach +# provider-adapter directly -- an endpoint with no credential to check does not +# belong on the public edge, and an address allowlist in front of it is a +# statement about the network, which is where it should be made. +location = /publish { + deny all; +} diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index cb451b6..94c60c2 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -1,5 +1,18 @@ -# The whole OAN stack in one file: registry, discovery, and the three adapters. -# Meant for a shared DEV deployment on a VM. +# The whole OAN stack in one file: registry, discovery, the three adapters, the +# telemetry stack and the public edge. +# +# Read it in tiers -- the banner comments below are the structure: +# +# registry registry-db, keycloak, registry +# discovery discovery-db, discovery +# adapters provider, network, exp +# observability hyperdx profile: observability +# edge nginx-proxy-manager profile: gateway +# +# The last two are behind profiles because neither is needed to exercise the +# stack, and HyperDX is the heaviest thing here. +# +# ---------------------------------------------------------------- bringing up # # 1. cp .env.example .env change every credential -- they are # shipped defaults @@ -8,6 +21,8 @@ # 3. python3 bin/setup.py keys, the three adapter entries, and # the adapter configs they mount # 4. docker compose up -d now the adapters +# 5. docker compose --profile gateway --profile observability up -d +# the edge and the telemetry stack # # Naming registry and discovery in step 2 is not tidiness. An adapter config # is a bind-mounted FILE, and Docker creates a DIRECTORY at any bind-mount @@ -15,20 +30,52 @@ # wedges that container on a directory it cannot parse and leaves a directory # where step 3 needs to write a file. # -# Nothing is built here. The adapter and discovery images are pulled from the -# tags named in .env. +# Nothing is built here. Every image is pulled from the tags named in .env. +# +# ------------------------------------------------------------- what is public +# +# Exactly one container publishes on a routable interface: Nginx Proxy +# Manager, on 80 and 443. Everything else -- both Postgres instances, +# Keycloak, the registry, discovery, the HyperDX UI, and NPM's own admin UI on +# 81 -- publishes on 127.0.0.1 and is reachable only through an SSH tunnel: # -# Ports bind to 127.0.0.1 by default. That is deliberate: this stack has an -# admin-console Keycloak and a registry whose write token any reader of -# .env.example can mint, so publishing it on a VM's public interface would -# hand over the whole network's identity records. Reach it over an SSH tunnel, -# or put a reverse proxy that terminates TLS and authenticates in front, and -# set BIND_ADDR only once something else is doing that job. +# ssh -L 81:127.0.0.1:81 -L 8080:127.0.0.1:8080 \ +# -L 8081:127.0.0.1:8081 -L 8085:127.0.0.1:8085 -N you@the-vm # -# There is deliberately NO provider here. The upstream API is run and exposed -# by whoever is testing -- see README.md. Nothing in this file knows the -# provider exists; the adapter learns its base URL from the registry at -# request time, so repointing it is a registry write and nothing more. +# The loopback binds are written literally rather than taken from a BIND_ADDR +# variable, which is what they used to be. One variable that moves the whole +# stack onto a public interface is a footgun once an edge exists to do that +# job deliberately for the one tier that should be reachable: behind those +# ports are a Keycloak whose admin password ships as a default and a registry +# whose write token anyone who has read .env.example can mint. +# +# ------------------------------------------------------------------- networks +# +# Two, and with a UI-configured proxy at the edge the split stops being +# decoration and becomes the actual control. NPM's "Forward Hostname" is a +# free-text field, so anyone with the admin password can point a public route +# at `registry` or `keycloak`. NPM sits only on oan-edge, where neither name +# resolves and neither address is routable -- the three adapters are the only +# containers it shares a network with. The adapters straddle both networks; +# everything else is internal-only. +# +# Neither is `internal: true`: the adapters and discovery fetch the Beckn spec +# from raw.githubusercontent.com at boot, the provider adapter calls an +# upstream API that lives outside this VM, and NPM has to reach Let's Encrypt. +# Cutting egress would break all three. + +networks: + oan-internal: + name: oan-internal + oan-edge: + name: oan-edge + +volumes: + registry-data: + discovery-data: + hyperdx-data: + npm-data: + npm-letsencrypt: x-adapter: &adapter # Pulled, never built. Set ADAPTER_IMAGE in .env to the tag published for @@ -36,21 +83,41 @@ x-adapter: &adapter image: ${ADAPTER_IMAGE} pull_policy: missing restart: unless-stopped - environment: &adapter-env - CONFIG_FILE: /app/config/adapter.yaml + # The adapters are the only services on oan-edge, and therefore the only + # ones NPM can reach. They still need oan-internal to read the registry and + # to call discovery, so they sit on both. + networks: [oan-internal, oan-edge] + +x-adapter-env: &adapter-env + CONFIG_FILE: /app/config/adapter.yaml + # Sent if the image's OpenTelemetry SDK picks these up, ignored if it does + # not -- which is the state this has not been verified in either direction, + # because the adapter image is pulled and its source is not in this repo. + # Nothing here depends on the answer: an unroutable or absent collector + # makes an OTLP exporter drop spans, not fail a request. What IS decided is + # the destination, so there is one place to change it. + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTLP_ENDPOINT:-http://hyperdx:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf services: - # ---------------------------------------------------------------- registry + # ========================================================================== + # registry -- who is on the network, their public keys, and which upstream + # API answers which capability. + # ========================================================================== registry-db: image: postgres:14 container_name: oan-registry-db restart: unless-stopped + networks: [oan-internal] environment: POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} volumes: - registry-data:/var/lib/postgresql/data + # No published port at all, not even on loopback: psql runs through + # `docker compose exec registry-db`, so publishing one would only widen + # the surface without adding a way in. healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] interval: 5s @@ -61,6 +128,7 @@ services: image: ghcr.io/sunbird-rc/sunbird-rc-keycloak:latest container_name: oan-keycloak restart: unless-stopped + networks: [oan-internal] volumes: - ./config/registry/imports:/opt/jboss/keycloak/imports environment: @@ -75,8 +143,10 @@ services: - KEYCLOAK_IMPORT=/opt/jboss/keycloak/imports/realm-export.json - PROXY_ADDRESS_FORWARDING=true ports: - - "${BIND_ADDR:-127.0.0.1}:${KEYCLOAK_PORT}:8080" - - "${BIND_ADDR:-127.0.0.1}:${KEYCLOAK_ADMIN_PORT}:9990" + # Loopback only. This is an admin console with an imported realm, and + # the tunnel is how you reach it. + - "127.0.0.1:${KEYCLOAK_PORT}:8080" + - "127.0.0.1:${KEYCLOAK_ADMIN_PORT}:9990" depends_on: registry-db: condition: service_healthy @@ -90,6 +160,27 @@ services: image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_VERSION} container_name: oan-registry restart: unless-stopped + # On oan-edge so NPM can resolve it, which is what makes a public proxy + # host for the registry possible at all. Read this before relying on it: + # + # POST /api/v1/Participant/search takes NO token -- that is the call a + # peer actually needs, and publishing it is defensible. Everything else + # under /api/v1/ is a write, and a write needs a Keycloak token. Those are + # unobtainable from outside today for one reason only: Keycloak publishes + # on 127.0.0.1. So the safety of this route rests on a decision made + # forty lines up, not on anything the route itself does -- publish + # Keycloak later and the write surface opens with it, silently. + # + # Which is why this host is not meant to be served bare. Two things go + # with it, both in README under "Adding a route": + # - an NPM Access List with Satisfy Any OFF, and + # - config/gateway/npm-advanced/registry.conf, which reduces the host to + # the two search endpoints and 403s the rest. + # + # If neither is in place, take oan-edge back off rather than leaving it + # attached "for later" -- an attached service is one form field away from + # being public. + networks: [oan-internal, oan-edge] volumes: # Schemas are read at startup, so a change here needs this service # restarted before the registry will honour it. @@ -131,7 +222,9 @@ services: - swagger_title=OAN Registry - logging.level.root=INFO ports: - - "${BIND_ADDR:-127.0.0.1}:${REGISTRY_PORT}:8081" + # Loopback only, and bin/setup.py talks to exactly this: it derives + # http://localhost:${REGISTRY_PORT} and runs on the VM. + - "127.0.0.1:${REGISTRY_PORT}:8081" depends_on: registry-db: condition: service_healthy @@ -148,14 +241,21 @@ services: retries: 60 start_period: 40s - # --------------------------------------------------------------- discovery + # ========================================================================== + # discovery -- catalogue search, and its own Postgres. + # ========================================================================== discovery-db: # pgvector rather than plain postgres: the discovery service's HNSW index # options arrived in 0.8. image: pgvector/pgvector:0.8.0-pg16 container_name: oan-discovery-db restart: unless-stopped + networks: [oan-internal] environment: + # Not read from .env, unlike the registry's pair, and so not covered by + # the credential rotation .env.example asks for. That is tolerable only + # because this database publishes no port and sits on an internal + # network; if it ever needs to be reachable, these move to .env first. POSTGRES_USER: discovery POSTGRES_PASSWORD: discovery POSTGRES_DB: discovery @@ -172,12 +272,25 @@ services: pull_policy: missing container_name: oan-discovery restart: unless-stopped + networks: [oan-internal] environment: DATABASE_URL: postgres://discovery:discovery@discovery-db:5432/discovery?sslmode=disable DATABASE_AUTO_MIGRATE: "true" APP_NETWORK_ID: ${APP_NETWORK_ID} SERVER_PORT: 8080 VALIDATION_SPEC_URL: ${BECKN_SPEC_URL} + + # Telemetry. The endpoint is the SDK's own variable, which is what this + # service reads (src/platform/config/config.go), so it needs no + # translation -- but the exporter stays `none` by default on purpose: + # in the current build OTEL_EXPORTER is parsed into config and nothing + # consumes it, and the only OpenTelemetry packages in go.mod are + # indirect. Setting it to otlp today emits nothing rather than failing; + # flip OTEL_EXPORTER=otlp in .env once the exporter is wired and the + # traces land in HyperDX with no other change here. + OTEL_EXPORTER: ${OTEL_EXPORTER:-none} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTLP_ENDPOINT:-http://hyperdx:4318} + OTEL_SERVICE_NAME: oan-discovery # config/common.yaml is baked into the image and is the reviewed default. # To override a setting, copy config/discovery/instance.yaml.example to # config/discovery/instance.yaml and uncomment the mount below -- compose @@ -186,12 +299,21 @@ services: # volumes: # - ./config/discovery/instance.yaml:/app/config/instance.yaml:ro ports: - - "${BIND_ADDR:-127.0.0.1}:${DISCOVERY_PORT}:8080" + # Loopback only. Discovery is reached by the network adapter over + # oan-internal; this publish exists for curl through the tunnel. + - "127.0.0.1:${DISCOVERY_PORT}:8080" depends_on: discovery-db: condition: service_healthy - # ---------------------------------------------------------------- adapters + # ========================================================================== + # adapters -- experience, network and provider. Same image, three configs. + # + # Their configs are bind-mounted FILES that do not exist until bin/setup.py + # renders them from the .tmpl beside them, which is what the step ordering + # above is about. + # ========================================================================== + # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, # so repointing it at a different upstream is a registry write, not a @@ -204,11 +326,12 @@ services: condition: service_healthy environment: <<: *adapter-env + OTEL_SERVICE_NAME: oan-provider-adapter volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro ports: - - "${BIND_ADDR:-127.0.0.1}:${PROVIDER_ADAPTER_PORT}:9200" + - "127.0.0.1:${PROVIDER_ADAPTER_PORT}:9200" # Verifies the caller, hands discovery on, re-signs as itself. network-adapter: @@ -221,15 +344,17 @@ services: condition: service_started environment: <<: *adapter-env + OTEL_SERVICE_NAME: oan-network-adapter volumes: - ./config/adapters/network.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-network.yaml:/app/config/routing-network.yaml:ro ports: - - "${BIND_ADDR:-127.0.0.1}:${NETWORK_ADAPTER_PORT}:9201" + - "127.0.0.1:${NETWORK_ADAPTER_PORT}:9201" # The caller, and the only one that takes unsigned requests: the experience # app is inside the trust boundary, so there is no network signature to - # check. This is what makes the stack testable with a plain curl. + # check. This is what makes the stack testable with a plain curl -- and it + # is also why the edge rate-limits this host and nothing else. exp-adapter: <<: *adapter container_name: oan-exp-adapter @@ -240,12 +365,172 @@ services: condition: service_started environment: <<: *adapter-env + OTEL_SERVICE_NAME: oan-exp-adapter volumes: - ./config/adapters/exp.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-exp.yaml:/app/config/routing-exp.yaml:ro ports: - - "${BIND_ADDR:-127.0.0.1}:${EXP_ADAPTER_PORT}:9202" + - "127.0.0.1:${EXP_ADAPTER_PORT}:9202" -volumes: - registry-data: - discovery-data: + # ========================================================================== + # observability -- HyperDX / ClickStack: OTLP ingest, ClickHouse, and the UI + # over it. + # + # docker compose --profile observability up -d + # + # Sizing: ClickHouse alone wants 2-4 GB. With this profile on, the VM needs + # 16 GB (t3.xlarge or m6i.xlarge); without it, 8 GB is workable. + # ========================================================================== + hyperdx: + # clickstack-local, not clickstack-all-in-one: local runs single-user with + # no team to create and no ingestion API key to mint, which is what makes + # `up -d` the whole setup step. It is also why this must stay on loopback + # -- there is no login in front of it. + image: clickhouse/clickstack-local:latest + container_name: oan-hyperdx + restart: unless-stopped + profiles: ["observability"] + networks: [oan-internal] + env_file: + # Optional. Present so an existing .env.docker keeps working; compose + # errors on a missing env_file unless it is declared this way. + - path: .env.docker + required: false + environment: + # Self-instrumentation off: the telemetry stack reporting on itself is + # noise in the same tables the stack under test writes to, and at debug + # level it is most of the volume. + OTEL_SDK_DISABLED: "true" + HYPERDX_LOG_LEVEL: error + HYPERDX_USAGE_STATS_ENABLED: "false" + HYPERDX_USAGE_STATS_COLLECTION_ENABLED: "false" + OTEL_LOG_LEVEL: none + OTEL_TRACES_EXPORTER: none + OTEL_METRICS_EXPORTER: none + OTEL_LOGS_EXPORTER: none + USAGE_STATS_ENABLED: "false" + ports: + # 8085, not the 8080 this image serves the UI on and not the 8081 it is + # usually published as: 8081 is REGISTRY_PORT here, and two services + # cannot claim one host port. + - "127.0.0.1:${HYPERDX_PORT:-8085}:8080" + # OTLP. Published on loopback only so a reverse tunnel can carry + # telemetry from a provider API running on someone's laptop; the + # in-stack senders reach hyperdx:4317/4318 over oan-internal and do not + # need these at all. + - "127.0.0.1:${OTLP_GRPC_PORT:-4317}:4317" + - "127.0.0.1:${OTLP_HTTP_PORT:-4318}:4318" + volumes: + - hyperdx-data:/var/lib/clickhouse + healthcheck: + # wget, not curl: this image has busybox. + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + # ========================================================================== + # edge -- Nginx Proxy Manager. + # + # docker compose --profile gateway up -d + # + # NPM rather than a hand-written nginx.conf because it owns the part that is + # genuinely tedious to do by hand -- ACME. It requests, installs and renews + # Let's Encrypt certificates from a UI rather than from a certbot invocation + # someone has to remember. In exchange, the routing table stops being a file + # in this repo and becomes rows in NPM's SQLite database under the npm-data + # volume: reviewable only by opening the UI, and restorable only from that + # volume. See README for the proxy hosts to create. + # ========================================================================== + nginx-proxy-manager: + image: jc21/nginx-proxy-manager:latest + container_name: oan-npm + restart: unless-stopped + profiles: ["gateway"] + + # oan-edge ONLY, and this is the whole security argument for putting a + # UI-configured proxy in front of this stack. + # + # NPM's "Forward Hostname" is a free-text field. Anyone with the admin + # password can type `registry`, `keycloak` or `discovery-db` into it. On + # this network none of those names resolve and none of those addresses are + # routable from here -- the three adapters are the only containers NPM + # shares a network with. So the blast radius of a wrong click, or of the + # admin UI being reached by someone who should not have it, is bounded to + # the tier that is meant to be public anyway. + # + # Putting NPM on oan-internal to "make things easier" would remove that + # bound entirely and put the registry's write API one form field away from + # the internet. Do not. + networks: [oan-edge] + + ports: + # Public. Both of these must be reachable from anywhere, not just from + # your address: Let's Encrypt validates HTTP-01 by fetching + # http:///.well-known/acme-challenge/... from its own servers, and + # those come from addresses you do not get to enumerate. An SG rule + # scoped to your IP will make certificate issuance fail with a challenge + # timeout. Use DNS-01 instead if 80 must stay closed -- see README. + - "0.0.0.0:${GATEWAY_HTTP_PORT:-80}:80" + - "0.0.0.0:${GATEWAY_HTTPS_PORT:-443}:443" + + # The admin UI, on loopback -- deliberately not the "publish it and + # restrict it in the firewall" arrangement. Two reasons. It ships with + # a known default login (admin@example.com / changeme) that is live from + # first boot until someone changes it, and it is the one surface here + # that can mint certificates and re-point every public route. A security + # group is a second system to keep in step with that; a loopback bind is + # not. Reach it over the tunnel: + # + # ssh -L 81:127.0.0.1:81 -N you@the-vm then http://127.0.0.1:81 + - "127.0.0.1:${NPM_ADMIN_PORT:-81}:81" + + volumes: + - npm-data:/data + - npm-letsencrypt:/etc/letsencrypt + + # Version-controlled nginx that NPM includes on its own. /data is a + # named volume and this bind mounts over one directory inside it, which + # works because the more specific mount wins. + # + # Only what is genuinely context-wide belongs here -- see the README for + # which of these NPM loads automatically and which is a paste job. + # + # NOT :ro, however much it wants to be. NPM's s6 init runs 50-ipv6.sh + # over everything under /data/nginx, which writes a .tmp beside each file + # and then chowns it. On a read-only mount both fail, the `prepare` + # service exits 1, s6 aborts the rest of the chain, and nginx never + # starts -- the container still reports Up, because s6 itself is alive, + # so the symptom is an empty reply on 81 and a healthcheck that never + # passes rather than anything that says "permissions". + # + # What NPM actually rewrites is `listen` directives, and neither file + # here has one, so the pass is a no-op on content. It does take + # ownership of the files, which is cosmetic and not tracked by git. + - ./config/gateway/npm-custom:/data/nginx/custom + + depends_on: + # Ordering only; the adapter image has no healthcheck. + # + # And ordering is not the whole problem. NPM writes a literal + # proxy_pass hostname per proxy host, which nginx resolves at reload and + # then caches -- so if you RECREATE an adapter (not merely restart it; + # a restart keeps the address) NPM keeps proxying to an address nothing + # answers on. The fix is one command, and it is worth knowing before you + # spend an afternoon on a 502: + # + # docker compose restart nginx-proxy-manager + exp-adapter: + condition: service_started + network-adapter: + condition: service_started + provider-adapter: + condition: service_started + + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:81/api/ || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s From a60c4beb9f4120e674ec80fc3aa466b37abb2f61 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:45:19 +0530 Subject: [PATCH 22/81] feat: add make targets for the stack lifecycle [#4] `make up` runs the five steps in the order they have to run in. The ordering is load-bearing rather than tidy: an adapter's config is a bind-mounted FILE that setup.py renders, and Docker creates a DIRECTORY at any bind-mount source that does not exist yet -- so starting an adapter first both wedges that container and leaves a directory where setup.py needs to write a file. Preflight checks .env, compose v2, python3 and the cryptography import before anything starts, because keycloak's healthcheck allows five minutes and a missing import would otherwise surface after that wait. down keeps volumes; destroy deletes them and requires the word typed, since npm-data is the only copy of every proxy host and certificate. up-core skips the gateway and hyperdx for a box that has neither a public port nor 16 GB. --- docker-deployment/Makefile | 45 ++++++ docker-deployment/bin/setup.py | 5 +- docker-deployment/bin/stack.sh | 281 +++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 docker-deployment/Makefile create mode 100755 docker-deployment/bin/stack.sh diff --git a/docker-deployment/Makefile b/docker-deployment/Makefile new file mode 100644 index 0000000..4a3b4ab --- /dev/null +++ b/docker-deployment/Makefile @@ -0,0 +1,45 @@ +# The front door. Every target is one line of delegation to bin/stack.sh, which +# is where the reasoning lives -- a Makefile is a bad place to explain why the +# startup order matters, and a good place to make the right order the shortest +# thing to type. +# +# make up the whole stack, in the order it has to start +# make up-core the same minus the gateway and hyperdx +# make down stop everything, keep the data +# make help the rest +# +# Anchored to this file's own directory rather than $(PWD), so `make -C +# docker-deployment up` works from the repo root. + +STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh + +.PHONY: help up up-core down destroy setup gateway observability restart-edge ps logs + +# Default target: running a bare `make` in a directory that can delete a +# Postgres volume should print the menu, not pick something from it. +help: + @$(STACK) --help + +# ---------------------------------------------------------------- the stack + +up: ; @$(STACK) up +up-core: ; @$(STACK) up-core +down: ; @$(STACK) down +destroy: ; @$(STACK) destroy + +# --------------------------------------------- the optional tiers, on their own +# +# `make up` already starts both. These are for starting one without the other, +# or restarting one after a config change. + +gateway: ; @$(STACK) gateway +observability: ; @$(STACK) observability + +# ------------------------------------------------------------------ the rest + +setup: ; @$(STACK) setup +restart-edge: ; @$(STACK) restart-edge +ps: ; @$(STACK) ps + +# `make logs` follows everything; `make logs SVC=registry` follows one service. +logs: ; @$(STACK) logs $(SVC) diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 835c574..4af3038 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -33,8 +33,9 @@ def load_dotenv(): """Read .env into the environment. - There is no Makefile here to source it first, and a real environment - variable wins so a one-off override still works: + Nothing sources .env before this runs -- `make up` shells straight out to + python3 -- and a real environment variable wins so a one-off override still + works: REGISTRY_PORT=9081 python3 bin/setup.py """ diff --git a/docker-deployment/bin/stack.sh b/docker-deployment/bin/stack.sh new file mode 100755 index 0000000..65a0509 --- /dev/null +++ b/docker-deployment/bin/stack.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# +# Bring the stack up in the order it has to come up in, and take it back down. +# +# bin/stack.sh up the whole stack, in the order it has to start +# bin/stack.sh up-core the same minus the gateway and hyperdx +# bin/stack.sh down stop everything, keep the data +# bin/stack.sh destroy stop everything, DELETE the data +# +# Everything here is also a make target: `make up`, `make down`. The Makefile is +# the front door; this file is where the reasoning lives. +# +# ------------------------------------------------------------ why a script +# +# The steps in `up` are not interchangeable and the failure mode of getting +# them wrong is not obvious. An adapter's config is a bind-mounted +# FILE that setup.py renders. Docker creates a DIRECTORY at any bind-mount +# source that does not exist yet -- so starting an adapter before step 2 both +# wedges that container on a directory it cannot parse AND leaves a directory +# sitting where step 2 needs to write a file. Recovering means `rm -rf`ing +# paths under config/adapters/ that look like they should be there. +# +# So the ordering is worth encoding once rather than remembering three times. + +set -euo pipefail + +# Every path in here is relative to the compose directory, and `docker compose` +# needs to find docker-compose.yml, so anchor to it rather than to $PWD. That +# makes `make -C docker-deployment up` work from anywhere. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# Both optional profiles, named explicitly. This matters for `down`: compose +# only acts on services whose profile is active, so a plain `docker compose +# down` leaves the gateway and hyperdx containers running and then reports +# success. Naming them on teardown is what makes "down" mean down. +PROFILES=(--profile gateway --profile observability) + +# ------------------------------------------------------------------ output + +# Steps are numbered in the output because the whole point of this script is +# that the order is load-bearing -- if it fails, you want to know at which one. +step() { printf '\n\033[1;36m==> [%s/%s] %s\033[0m\n' "$1" "$2" "$3"; } +info() { printf ' %s\n' "$1"; } +warn() { printf '\033[1;33mwarning: %s\033[0m\n' "$1" >&2; } +die() { printf '\033[1;31mstack: %s\033[0m\n' "$1" >&2; exit 1; } + +# -------------------------------------------------------------- preflight + +# Checked before anything starts, not when it is first needed. Step 1 blocks on +# Keycloak's healthcheck, which is 30 retries at 10s -- so a missing python3 or +# an unimportable `cryptography` would surface several minutes in, after a wait +# that had nothing to do with the problem. These take a millisecond each. +preflight() { + [ -f .env ] || die ".env is missing -- cp .env.example .env, then change every credential in it" + + docker compose version >/dev/null 2>&1 \ + || die "docker compose (v2) is not available -- this needs the plugin, not docker-compose" + + command -v python3 >/dev/null 2>&1 \ + || die "python3 is not installed -- bin/setup.py needs it" + + python3 -c 'import cryptography' >/dev/null 2>&1 \ + || die "the python 'cryptography' package is missing -- pip install cryptography" +} + +# ------------------------------------------------------------------- up + +# The full stack, in the order the compose file's own header documents. Five +# steps rather than three: the gateway and hyperdx are behind profiles, which +# means they are opt-in for compose, but "opt-in" and "not part of bringing the +# stack up" are different claims and only the first one is true. +# +# `up-core` below is the same thing minus steps 4 and 5, for when you want +# neither a public port nor ClickHouse's memory. +up() { + preflight + up_registry_tier 5 + up_setup 5 + up_adapters 5 + + # NPM. This is the step that makes the VM reachable from the internet -- + # 0.0.0.0:80 and :443, deliberately not scoped, because Let's Encrypt + # validates HTTP-01 from its own servers. + step 4 5 "nginx-proxy-manager -- the public edge (80 and 443, all interfaces)" + docker compose --profile gateway up -d nginx-proxy-manager + + # ClickStack. Heaviest thing here by a wide margin: ClickHouse alone wants + # 2-4 GB, which is what takes this VM from 8 GB to 16 GB. + step 5 5 "hyperdx -- ClickStack (OTLP ingest, ClickHouse, UI)" + docker compose --profile observability up -d hyperdx + + done_banner +} + +# Steps 1-3 only. Nothing publishes on a routable interface and nothing needs +# 16 GB -- this is the stack you can actually exercise, which is the reason the +# profiles exist in the first place. +up_core() { + preflight + up_registry_tier 3 + up_setup 3 + up_adapters 3 + done_banner +} + +# --- the shared steps, so `up` and `up-core` cannot drift apart ------------- +# +# Each takes the step total so the numbering reads correctly in both. + +# Naming registry and discovery also starts registry-db, keycloak and +# discovery-db: all are depends_on with condition: service_healthy, so compose +# blocks here until they pass their healthchecks rather than racing ahead. +# +# discovery belongs in this step and not in step 3. It would be dragged in +# anyway by network-adapter's depends_on, but then a discovery-db that failed +# to come up would surface as an adapter problem three steps later. +# +# No --wait. It would only add a second wait on the registry's own healthcheck, +# and setup.py already polls with an error message that says what to check. +up_registry_tier() { + step 1 "$1" "registry and discovery (also starts registry-db, keycloak, discovery-db)" + info "keycloak's healthcheck allows up to 5 minutes on a cold volume" + docker compose up -d registry discovery +} + +# Generates the adapter keypairs, registers the three adapter identities, and +# renders config/adapters/{provider,network,exp}.yaml from the .tmpl files +# beside them. Safe to re-run: keys come from keys/keys.json once it exists, +# and participants already registered are left alone. +up_setup() { + step 2 "$1" "bin/setup.py -- keys, adapter identities, adapter configs" + python3 bin/setup.py +} + +# Only now do the bind-mounted config files exist. +up_adapters() { + step 3 "$1" "adapters (provider, network, exp)" + docker compose up -d provider-adapter network-adapter exp-adapter +} + +done_banner() { + printf '\n\033[1;32m==> stack is up\033[0m\n' + docker compose "${PROFILES[@]}" ps + cat <<'NEXT' + + Everything except NPM's 80 and 443 is bound to 127.0.0.1 on this host. + From your laptop: + + ssh -L 81:127.0.0.1:81 -L 8080:127.0.0.1:8080 \ + -L 8081:127.0.0.1:8081 -L 8085:127.0.0.1:8085 -N you@the-vm + + If the gateway is running, its admin UI is on http://127.0.0.1:81 and still + has its shipped login (admin@example.com / changeme) until you change it. + That account can mint certificates and re-point every public route -- change + it before creating anything. + +NEXT +} + +# ----------------------------------------------------------------- down + +# Containers and networks go; named volumes stay. So the registry's Postgres +# data, discovery's data, and -- the one that would actually hurt -- npm-data, +# which is the ONLY copy of every proxy host and Let's Encrypt certificate, +# all survive. `up` after this is fast and lands where you left off. +down() { + step 1 1 "stopping everything, keeping the data" + docker compose "${PROFILES[@]}" down --remove-orphans + info "volumes kept. 'make destroy' is the one that deletes them." +} + +# -------------------------------------------------------------- destroy + +# The asymmetry with `down` is deliberate: this is not recoverable, and one of +# the things it deletes was never in git to begin with. +destroy() { + cat <<'WARN' +This deletes every named volume in the project: + + npm-data every NPM proxy host and Let's Encrypt certificate. NPM + keeps its routing table in a SQLite database in this + volume and nowhere else -- there is no export, and the + admin account has no reset flow. If you have not backed + it up, the click-through starts over. + registry-data the registry's Postgres: participants, keys, schemas. + discovery-data the discovery catalogue. + hyperdx-data collected telemetry. + +keys/keys.json is NOT deleted, and should not be -- it is what lets setup.py +re-register the adapters under their existing identities on the next `up`. + +WARN + if [ "${FORCE:-}" != "1" ]; then + [ -t 0 ] || die "not a terminal -- re-run as FORCE=1 make destroy if you mean it" + read -r -p "Type 'destroy' to confirm: " reply + [ "$reply" = "destroy" ] || die "aborted" + fi + + step 1 1 "removing containers, networks and volumes" + docker compose "${PROFILES[@]}" down -v --remove-orphans +} + +# ------------------------------------------------------- optional tiers + +# Separate targets rather than part of `up` because neither is needed to +# exercise the stack, and hyperdx (ClickHouse) alone wants 2-4 GB. +gateway() { + preflight + step 1 1 "nginx-proxy-manager -- publishes 80 and 443 on ALL interfaces" + docker compose --profile gateway up -d nginx-proxy-manager + cat <<'NEXT' + + The admin UI is on loopback, and ships with a live default login. Tunnel in + and change it before creating anything: + + ssh -L 81:127.0.0.1:81 -N you@the-vm then http://127.0.0.1:81 + +NEXT +} + +observability() { + preflight + step 1 1 "hyperdx -- ClickStack (OTLP ingest, ClickHouse, UI)" + docker compose --profile observability up -d hyperdx + info "UI on 127.0.0.1:8085. There is no login in front of it -- keep it on loopback." +} + +# ----------------------------------------------------------------- misc + +# NPM writes a literal proxy_pass hostname per proxy host, which nginx resolves +# at reload and then caches. RECREATE an adapter -- not merely restart it, a +# restart keeps the address -- and NPM goes on proxying to an address nothing +# answers on. This is the fix, and it is worth having as a target because the +# symptom is a bare 502 that looks like the adapter is down. +restart_edge() { + step 1 1 "restarting nginx-proxy-manager to re-resolve adapter addresses" + docker compose --profile gateway restart nginx-proxy-manager +} + +# Just step 2. Re-run it after editing a .tmpl, or to re-render configs that +# were deleted. It is idempotent, so this is always safe. +setup() { + preflight + step 1 1 "bin/setup.py" + python3 bin/setup.py +} + +usage() { + cat <<'USAGE' +bin/stack.sh + + up the whole stack: registry+discovery -> setup.py -> adapters + -> gateway (public, 80/443) -> hyperdx (wants 16 GB) + up-core steps 1-3 only. Nothing public, no ClickHouse. + down stop everything, keep the data + destroy stop everything and DELETE every volume + setup re-run bin/setup.py only + gateway start nginx-proxy-manager on its own (public, 80/443) + observability start hyperdx on its own + restart-edge restart NPM after recreating an adapter (fixes a 502) + ps docker compose ps + logs [service] docker compose logs -f + +USAGE +} + +case "${1:-}" in + up) up ;; + up-core) up_core ;; + down) down ;; + destroy) destroy ;; + setup) setup ;; + gateway) gateway ;; + observability) observability ;; + restart-edge) restart_edge ;; + ps) docker compose "${PROFILES[@]}" ps ;; + logs) shift; docker compose "${PROFILES[@]}" logs -f "$@" ;; + ""|-h|--help) usage ;; + *) usage; die "unknown command: $1" ;; +esac From 4cb2b73038c3d160e0397cac574e3cc08130df42 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:45:42 +0530 Subject: [PATCH 23/81] feat: add ubuntu VM bootstrap script [#4] Installs docker from docker's own apt repository rather than the docker.io package or the snap -- the snap runs confined, and bind-mounting a file out of a home directory, which every adapter config does, fails under it in ways that read as file-not-found. Takes cryptography from apt rather than pip: Ubuntu 24.04 marks the system python externally-managed, so pip refuses without --break-system-packages, and setup.py needs that import. Checks RAM and disk up front. An undersized box fails by having the OOM killer take out a Postgres mid-write, which surfaces as corruption rather than as an out-of-memory error. --- docker-deployment/bin/bootstrap-ubuntu.sh | 121 ++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100755 docker-deployment/bin/bootstrap-ubuntu.sh diff --git a/docker-deployment/bin/bootstrap-ubuntu.sh b/docker-deployment/bin/bootstrap-ubuntu.sh new file mode 100755 index 0000000..e682a81 --- /dev/null +++ b/docker-deployment/bin/bootstrap-ubuntu.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# Everything an Ubuntu VM needs before `make up` will run. Idempotent -- safe to +# re-run, and safe to run on a box that already has some of this. +# +# curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts//docker-deployment/bin/bootstrap-ubuntu.sh | bash +# +# or, once the repo is cloned: +# +# bin/bootstrap-ubuntu.sh +# +# It does NOT clone the repo, write .env, or start anything. Those need +# decisions -- which branch, which credentials -- that do not belong in a +# script piped from the internet. + +set -euo pipefail + +say() { printf '\n\033[1;36m==> %s\033[0m\n' "$1"; } +info() { printf ' %s\n' "$1"; } +die() { printf '\033[1;31mbootstrap: %s\033[0m\n' "$1" >&2; exit 1; } + +[ "$(id -u)" -ne 0 ] || die "run as a normal user, not root -- this uses sudo where it needs to" +command -v apt-get >/dev/null || die "not a Debian/Ubuntu system" + +# ---------------------------------------------------------------- sizing + +# Checked rather than assumed, because the failure mode of an undersized box is +# the OOM killer taking out a Postgres mid-write, which surfaces as data +# corruption rather than as "out of memory". +say "checking this VM against what the stack needs" +mem_gb=$(( $(awk '/MemTotal/{print $2}' /proc/meminfo) / 1024 / 1024 )) +disk_gb=$(df -BG --output=avail / | tail -1 | tr -dc '0-9') +info "RAM ${mem_gb} GB (8 GB minimum, 16 GB if you run the observability profile)" +info "disk ${disk_gb} GB free (20 GB minimum -- the images alone are ~6 GB)" +[ "$mem_gb" -ge 7 ] || info "WARNING: under 8 GB. Two Postgres, two JVMs and three adapters will not fit." +[ "$disk_gb" -ge 20 ] || info "WARNING: under 20 GB free. A full disk corrupts the Docker VM rather than erroring cleanly." + +# ------------------------------------------------------------------ apt + +say "apt packages" +sudo apt-get update -qq +# python3-cryptography from apt rather than pip: Ubuntu 24.04 marks the system +# python as externally-managed (PEP 668), so `pip install cryptography` refuses +# without --break-system-packages. The apt build is the same library. +sudo apt-get install -y -qq \ + ca-certificates curl gnupg git make python3 python3-cryptography +info "git, make, python3, python3-cryptography" + +# --------------------------------------------------------------- docker + +if command -v docker >/dev/null && docker compose version >/dev/null 2>&1; then + say "docker already present" + info "$(docker --version)" + info "$(docker compose version)" +else + # Docker's own apt repo, not the `docker.io` package and not snap. The + # distro package lags, and the snap runs confined -- bind mounts out of a + # home directory, which this stack does for every adapter config, fail + # under it in ways that read as file-not-found. + say "installing docker engine from docker's apt repository" + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes + sudo chmod a+r /etc/apt/keyrings/docker.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ +https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null + sudo apt-get update -qq + sudo apt-get install -y -qq \ + docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + info "$(docker --version)" +fi + +say "docker group" +if id -nG "$USER" | tr ' ' '\n' | grep -qx docker; then + info "$USER is already in the docker group" +else + sudo usermod -aG docker "$USER" + info "$USER added to the docker group" + info "LOG OUT AND BACK IN before docker works without sudo -- group" + info "membership is read at login, so this shell still cannot use it." +fi + +say "enabling docker at boot" +sudo systemctl enable --now docker >/dev/null 2>&1 || true +info "$(systemctl is-active docker 2>/dev/null || echo unknown)" + +# ------------------------------------------------------------------ ufw + +# Only touched if it is already running. Turning a firewall on for someone is +# a good way to end a session, and on EC2 the security group is the control +# that matters anyway. +if command -v ufw >/dev/null && sudo ufw status 2>/dev/null | grep -q "^Status: active"; then + say "ufw is active -- opening what the edge needs" + sudo ufw allow 80/tcp >/dev/null + sudo ufw allow 443/tcp >/dev/null + info "80 and 443 allowed. Everything else in this stack binds 127.0.0.1" + info "and is reached over the SSH tunnel, so nothing further is needed." +else + say "ufw not active -- leaving it alone" + info "On EC2 the security group is the real control. 80 and 443 must be" + info "open to 0.0.0.0/0, not to your address: Let's Encrypt validates" + info "HTTP-01 from its own servers." +fi + +say "done" +cat <<'NEXT' + + If this added you to the docker group, log out and back in now. + + Then: + + git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git + cd helmcharts/docker-deployment + cp .env.example .env && nano .env # change every credential + make up + + `make up` includes the gateway and hyperdx. Use `make up-core` for neither. + See CERTIFICATES.md before requesting a certificate. + +NEXT From 7dbc6dbfaa669caf0469a9e25ec7e8bb7904c7a2 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:45:43 +0530 Subject: [PATCH 24/81] docs: document the local and VM certificate flows [#4] Two mechanisms, split by whether the name is reachable from the internet. Let's Encrypt cannot certify a loopback name -- 127.0.0.1.sslip.io resolves correctly, to 127.0.0.1, which is exactly why their validators cannot reach it -- so local work needs mkcert and a private CA. Includes the failure table, because every symptom in this area points away from the cause: a CA present in the keychain but carrying no trust setting verifies fine under `curl --cacert` and fails in every browser, and a host with no certificate attached fails as an SNI alert rather than as anything mentioning certificates. --- docker-deployment/CERTIFICATES.md | 101 ++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docker-deployment/CERTIFICATES.md diff --git a/docker-deployment/CERTIFICATES.md b/docker-deployment/CERTIFICATES.md new file mode 100644 index 0000000..eb46661 --- /dev/null +++ b/docker-deployment/CERTIFICATES.md @@ -0,0 +1,101 @@ +# Certificates + +Two different mechanisms, and which one you need depends on whether the name +you are certifying is reachable from the public internet. + +| | local (this laptop) | the VM | +|----------------|--------------------------|-------------------------------| +| issuer | mkcert, a private CA | Let's Encrypt, a public CA | +| trusted by | only machines you set up | everything | +| validation | none -- you own the CA | HTTP-01, inbound from LE | +| works offline | yes | no | + +The dividing line is not preference. A public CA will only certify a name it +can verify from the outside, and a loopback address is definitionally outside +that. `127.0.0.1.sslip.io` resolves correctly -- to `127.0.0.1` -- which is +precisely why Let's Encrypt cannot validate it: their servers resolve that name +and connect to their own loopback. They also refuse by policy to issue for +names in reserved IP space, so this fails as a rejection and not a timeout. + +## Local, with mkcert + + brew install mkcert + +### 1. Create and trust the CA + + mkcert -install + +Run this in a REAL terminal window. It needs an admin password, and a sudo +prompt with no TTY fails silently -- which leaves the CA generated but not +trusted, and every certificate signed by it then fails in every browser while +looking perfectly well-formed in `openssl`. + +Verify before going further. This is the whole gate: + + security verify-cert -c "$(mkcert -CAROOT)/rootCA.pem" + +`Cert Verify Result: Success` and nothing else. `CSSMERR_TP_NOT_TRUSTED` means +the CA is present but carries no trust setting, which is the same as absent. + +### 2. Generate the leaf + + mkdir -p ~/oan-local-certs && cd ~/oan-local-certs + mkcert -cert-file oan-local.pem -key-file oan-local-key.pem \ + "127.0.0.1.sslip.io" "*.127.0.0.1.sslip.io" \ + "oan.test" "*.oan.test" localhost 127.0.0.1 ::1 + +Name everything up front; adding one later means regenerating and re-uploading. + +The sslip.io forms are worth having over the `.test` ones: sslip.io resolves +`.127.0.0.1.sslip.io` to `127.0.0.1` on its own, so those names need +no /etc/hosts entry. X.509 wildcards match one level only -- `exp.127.0.0.1.sslip.io` +is covered, `a.b.127.0.0.1.sslip.io` is not. + +### 3. Upload, attach, restart + +NPM -> SSL Certificates -> Add Certificate -> **Custom**. Key first, then cert. + +Then per proxy host: Edit -> SSL -> select it -> Force SSL -> Save. This is not +global; a host you forget serves plain HTTP and reports no error. + +Then QUIT the browser -- fully, not just the window. Chromium and Safari read +root certificates at process start, so a reload after `mkcert -install` shows +the old answer. + +## On the VM, with Let's Encrypt + +1. Elastic IP attached. Not optional with sslip.io: the hostname CONTAINS the + address, so a stop/start invalidates every proxy host and every certificate + at once, rather than needing one A record updated. + +2. Security group: 80 and 443 open to `0.0.0.0/0`. Not to your address -- + HTTP-01 validation arrives from Let's Encrypt's own servers, whose addresses + you do not get to enumerate. Scoping it to yourself fails with a challenge + timeout that reads like a DNS problem. + +3. `make gateway`, then create the proxy host with `..sslip.io`. + No DNS record to create -- that is the entire point of sslip.io here. + +4. Confirm it answers over plain HTTP first. Let's Encrypt allows 5 failed + validations per hostname per hour, and debugging a misconfigured route + through the certificate flow is how you spend them. + +5. SSL tab -> Request a new SSL Certificate -> HTTP Validation -> Force SSL. + +DNS-01 is unavailable with sslip.io -- you do not control that zone, so certbot +cannot write the TXT record. It is the better option on a domain you do own: it +needs no inbound reachability at all, and it is the only way to get a wildcard. + +Renewal uses whatever method issued the certificate, so an SG rule that was +only temporarily correct fails in sixty days with nothing to announce it. + +## Reading the failure + +| symptom | cause | +|---|---| +| `CSSMERR_TP_NOT_TRUSTED` | CA not in the trust store -- step 1 | +| Safari: `"" certificate is not trusted` | same, named after the cert's first SAN rather than the site | +| Brave: `Not Secure`, https struck through | same, with less detail. Safari's message is the useful one | +| `curl` verify=0 but browser red | `--cacert` bypasses the trust store; it proves the chain, not the trust | +| `tlsv1 unrecognized name`, SNI alert 112 | no 443 server block for that name -- the host has no certificate attached | +| cert fine, browser still red | browser not restarted since `mkcert -install` | From b852e8508551073a36dd3cc1ecd5e717b52e7b30 Mon Sep 17 00:00:00 2001 From: ameersohel45 <136956876+ameersohel45@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:32:54 +0530 Subject: [PATCH 25/81] fix: emit a location only when the provider echoed one [OpenAgriNet/network-adapter#4] If the provider answered without its location echo, JSONata dropped the undefined values inside the array and this emitted "coordinates": [] -- an invalid GeoJSON Point, which the adapter then signed and delivered. Nothing downstream could detect it: the payload was well formed and spec-shaped, just describing a point that does not exist. Absent is honest; empty is a lie in the shape of an answer. This deployment serves its own copy of the mapping, deliberately, so the file a reviewer reads is the file the adapter fetches -- which also means it does not pick the fix up from anywhere else. Brought level with the adapter's reference copy, where it was fixed in review of network-adapter PR #2. Mapping only. The other findings from that review are in Go and reach this deployment when the adapter image is next published. --- .../mappings/mausamgram/weather-observation.select.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml index 0b1fe69..1baf294 100644 --- a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml +++ b/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml @@ -234,7 +234,13 @@ response: | "sourceId": "mausamgram", "sourceName": "IMD Mausamgram NWP" }, - "location": { + /* Emitted only when the provider echoed both coordinates. + JSONata drops undefined values inside an array, so a + provider that answered without its location echo would + otherwise produce "coordinates": [] -- an invalid Point, + signed and delivered. Absent is honest; empty is a lie + in the shape of an answer. */ + "location": $exists($lat) and $exists($lon) ? { "type": "Point", "coordinates": [$lon, $lat] }, From 89f52556dcc3f635e36f8ef876d99b0c3e94eb8b Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:39:20 +0530 Subject: [PATCH 26/81] fix: make the hyperdx healthcheck reach the service it probes [#4] busybox resolves localhost to ::1 first and this image binds v4 only, so the check failed with "Connection refused" against [::1]:8080 while the UI on 8085 served 200 throughout. A container reporting unhealthy while working sends you to ClickHouse, which is the one place the problem was not. Also widens the window. ClickHouse creates its system tables on a cold volume, which takes minutes rather than the 80 seconds the previous retries/start_period pair allowed, and Docker does not re-evaluate once a container has failed past retries -- so a window that is merely tight leaves a label that is permanently wrong. --- docker-deployment/docker-compose.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 94c60c2..e8da0a1 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -424,11 +424,22 @@ services: - hyperdx-data:/var/lib/clickhouse healthcheck: # wget, not curl: this image has busybox. - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health"] + # + # 127.0.0.1 and not localhost, which is not a style preference. busybox + # resolves localhost to ::1 first, and this image binds v4 only -- so the + # check fails with "Connection refused" against [::1]:8080 while the UI + # on 8085 serves perfectly well. An unhealthy container that is in fact + # working is worse than either state on its own, because it sends you + # looking at ClickHouse. + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/health"] interval: 10s timeout: 5s - retries: 5 - start_period: 30s + # ClickHouse creates its system tables on a cold volume, which takes + # minutes rather than the 80s the previous 5/30s pair allowed. Docker + # does not re-evaluate once a container has failed past `retries`, so a + # window that is merely tight leaves a permanent wrong label. + retries: 10 + start_period: 180s # ========================================================================== # edge -- Nginx Proxy Manager. From cef3aa30e37358a58aa05bb53a5ba9fbbeed2c74 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:00:00 +0530 Subject: [PATCH 27/81] feat: run both mock upstreams in the stack [OpenAgriNet/network-adapter#4] A select had nothing to answer it out of the box. Whoever brought the stack up had to run a provider API themselves and expose it through a tunnel, which made the first end-to-end run depend on the one piece that was not here. Both mocks are in the compose file now, on oan-internal and published on loopback only for looking at directly while debugging. The provider adapter reaches them by service name, so the registry rows point at http://mockimd:9100 and http://mockagmarknet:9101 rather than at anything that has to exist outside. Sources are in mocks/, byte-identical to the ones the oan-local stack has been tested against, and they are pulled as published images like everything else here -- nothing in this stack builds. mocks/README.md carries the build commands and, more usefully, what each one deliberately gets wrong: mockagmarknet answers 401 without its token, sends prices as strings with Title-Case spaced keys, and leaves the minimum and maximum off its last record. Those are the cases the mappings exist to handle, so a mock that answered cleanly would test nothing. MANDI_TOKEN reaches the provider adapter as an environment variable and the mock as a flag, from the same .env value. The adapter config names the variable, never the value. --- docker-deployment/docker-compose.yml | 51 ++++++ docker-deployment/mocks/README.md | 43 +++++ .../mocks/mockagmarknet/Dockerfile | 11 ++ docker-deployment/mocks/mockagmarknet/go.mod | 3 + docker-deployment/mocks/mockagmarknet/main.go | 127 +++++++++++++++ docker-deployment/mocks/mockimd/Dockerfile | 11 ++ docker-deployment/mocks/mockimd/go.mod | 3 + docker-deployment/mocks/mockimd/main.go | 147 ++++++++++++++++++ 8 files changed, 396 insertions(+) create mode 100644 docker-deployment/mocks/README.md create mode 100644 docker-deployment/mocks/mockagmarknet/Dockerfile create mode 100644 docker-deployment/mocks/mockagmarknet/go.mod create mode 100644 docker-deployment/mocks/mockagmarknet/main.go create mode 100644 docker-deployment/mocks/mockimd/Dockerfile create mode 100644 docker-deployment/mocks/mockimd/go.mod create mode 100644 docker-deployment/mocks/mockimd/main.go diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index e8da0a1..8cb3d3d 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -314,6 +314,50 @@ services: # above is about. # ========================================================================== + # ========================================================================== + # mock upstreams -- stand-ins for the real provider APIs, so the stack can be + # exercised without their credentials. + # + # Pulled like everything else. The sources are in mocks/ to be built and + # published once, not built here -- see mocks/README.md. + # + # oan-internal only: an upstream is called BY the provider adapter and is + # never reached from outside, so NPM has no business seeing it. The published + # ports are for looking at the mock directly while debugging. + # + # Both reproduce the awkward parts of the services they stand in for on + # purpose. A tidy mock would let a mapping pass here and fail in the real + # deployment. + # ========================================================================== + mockimd: + image: ${MOCKIMD_IMAGE} + pull_policy: missing + container_name: oan-mockimd + restart: unless-stopped + networks: [oan-internal] + command: + - "-addr=:9100" + # How many forecast days it answers with. The mapping reads however many + # arrive, so this is the knob for checking that it does. + - "-days=${MOCKIMD_DAYS:-3}" + ports: + - "127.0.0.1:${MOCKIMD_PORT}:9100" + + mockagmarknet: + image: ${MOCKAGMARKNET_IMAGE} + pull_policy: missing + container_name: oan-mockagmarknet + restart: unless-stopped + networks: [oan-internal] + command: + - "-addr=:9101" + # It answers 401 without this token, which is what proves the adapter + # sent one rather than the call happening to work anyway. + - "-token=${MANDI_TOKEN}" + - "-days=${MOCKAGMARKNET_DAYS:-2}" + ports: + - "127.0.0.1:${MOCKAGMARKNET_PORT}:9101" + # Verifies the caller, calls the upstream provider, answers synchronously. # It has no provider address of its own: it reads the ProviderSchema row, # so repointing it at a different upstream is a registry write, not a @@ -324,9 +368,16 @@ services: depends_on: registry: condition: service_healthy + mockimd: + condition: service_started + mockagmarknet: + condition: service_started environment: <<: *adapter-env OTEL_SERVICE_NAME: oan-provider-adapter + # The mandi upstream's credential. Named here and read at call time, so + # it is never in a config file or in the registry. + MANDI_TOKEN: ${MANDI_TOKEN} volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro diff --git a/docker-deployment/mocks/README.md b/docker-deployment/mocks/README.md new file mode 100644 index 0000000..e19165f --- /dev/null +++ b/docker-deployment/mocks/README.md @@ -0,0 +1,43 @@ +# Mock providers + +Two stand-in upstreams, so the stack can be exercised end to end without +credentials for the real services. + +| mock | stands in for | port | auth | +|---|---|---|---| +| `mockimd` | IMD Mausamgram NWP | 9100 | basic | +| `mockagmarknet` | Agmarknet Vistaar | 9101 | token as a query parameter | + +## Why the sources are here + +The compose file **pulls** every image and builds nothing, so these are not +built by `make up`. They are here to be built and published once, and then +pulled like everything else: + + docker build -t ghcr.io//oan-mockimd:latest mocks/mockimd + docker build -t ghcr.io//oan-mockagmarknet:latest mocks/mockagmarknet + docker push ghcr.io//oan-mockimd:latest + docker push ghcr.io//oan-mockagmarknet:latest + +Then set `MOCKIMD_IMAGE` and `MOCKAGMARKNET_IMAGE` in `.env`. + +## What they deliberately get wrong + +A mock that answered tidily would let a mapping pass here and fail against the +real service, so both reproduce the awkward parts on purpose. + +`mockimd` answers `fcstday1..N` with the count coming from `-days`, so a mapping +that hardcodes five days is caught. Forecasts derive from the requested point, +so a wrong lat/lon shows up as wrong numbers rather than passing silently. + +`mockagmarknet` answers a **bare JSON array** whose records use **Title Case +keys containing spaces** — `Modal Price`, `Arrival Date` — with **prices as +strings** and dates as `dd-MM-yyyy`. It requires the token as a query +parameter and answers 401 without it, which is what proves the adapter sent +one. Its last record reports no minimum or maximum, as the real data +sometimes does, so a mapping is forced to distinguish "not reported" from +"zero". Prices derive from the requested market and commodity codes, so a +wrong code is visible in the answer. + +Neither reproduces the real services' error bodies or credentials. What the +real ones do on no-data, rate limits or auth failure is still unobserved. diff --git a/docker-deployment/mocks/mockagmarknet/Dockerfile b/docker-deployment/mocks/mockagmarknet/Dockerfile new file mode 100644 index 0000000..42a8747 --- /dev/null +++ b/docker-deployment/mocks/mockagmarknet/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.26.1-bookworm AS builder +WORKDIR /src +COPY go.mod ./ +COPY main.go ./ +RUN CGO_ENABLED=0 go build -o mockagmarknet . + +FROM cgr.dev/chainguard/wolfi-base:latest +WORKDIR /app +COPY --from=builder /src/mockagmarknet . +EXPOSE 9101 +ENTRYPOINT ["./mockagmarknet"] diff --git a/docker-deployment/mocks/mockagmarknet/go.mod b/docker-deployment/mocks/mockagmarknet/go.mod new file mode 100644 index 0000000..2abe3ef --- /dev/null +++ b/docker-deployment/mocks/mockagmarknet/go.mod @@ -0,0 +1,3 @@ +module mockagmarknet + +go 1.22.2 diff --git a/docker-deployment/mocks/mockagmarknet/main.go b/docker-deployment/mocks/mockagmarknet/main.go new file mode 100644 index 0000000..98ecc42 --- /dev/null +++ b/docker-deployment/mocks/mockagmarknet/main.go @@ -0,0 +1,127 @@ +// Command mockagmarknet stands in for Agmarknet's Vistaar API during local +// end-to-end runs. +// +// It answers the shape the real service does: a bare JSON array of records with +// Title Case keys containing spaces and prices as strings. Both of those are +// awkward, and reproducing them is the point -- a mock that returned tidy +// camelCase numbers would let a mapping pass here and fail against the real +// thing. +// +// It requires the token as a query parameter, which is how that API +// authenticates, so the adapter's query auth path is exercised rather than +// skipped. Prices are derived from the requested market and commodity codes, so +// a wrong code shows up as wrong numbers instead of passing silently. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "hash/fnv" + "log" + "net/http" + "os" + "time" +) + +// record is one market's report for one day, keyed exactly as Agmarknet keys it. +type record struct { + Grade string `json:"Grade"` + Group string `json:"Group"` + State string `json:"State"` + Market string `json:"Market"` + Variety string `json:"Variety"` + District string `json:"District"` + Commodity string `json:"Commodity"` + MaxPrice string `json:"Max Price,omitempty"` + MinPrice string `json:"Min Price,omitempty"` + PriceUnit string `json:"Price Unit"` + ModalPrice string `json:"Modal Price"` + ArrivalDate string `json:"Arrival Date"` +} + +func main() { + addr := flag.String("addr", ":9101", "address to listen on") + token := flag.String("token", "local-mandi-token", "token the query must carry") + days := flag.Int("days", 2, "how many daily records to answer with") + flag.Parse() + + http.HandleFunc("/v1/fetch-agmarknet-vistaar", func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + log.Printf("%s %s?%s", r.Method, r.URL.Path, r.URL.RawQuery) + + if query.Get("token") != *token { + // The real API answers 401 for a bad token. Worth reproducing: it is + // what proves the adapter sent one at all. + http.Error(w, `{"message":"invalid token"}`, http.StatusUnauthorized) + return + } + for _, required := range []string{"statecode", "districtcode", "commoditycode", "from_date", "to_date"} { + if query.Get(required) == "" { + http.Error(w, fmt.Sprintf(`{"message":"missing %s"}`, required), http.StatusBadRequest) + return + } + } + + from, err := time.Parse("02-01-2006", query.Get("from_date")) + if err != nil { + // dd-MM-yyyy, not ISO. A mapping that forgets to convert lands here. + http.Error(w, `{"message":"from_date must be dd-MM-yyyy"}`, http.StatusBadRequest) + return + } + + commodity := query.Get("commoditycode") + market := query.Get("marketcode") + if market == "" { + // Without a market code the real API widens to the district, so the + // answer names the district rather than one market. + market = "district-" + query.Get("districtcode") + } + base := 1500 + int(hash(market+commodity)%800) + + records := make([]record, 0, *days) + for day := 0; day < *days; day++ { + date := from.AddDate(0, 0, day) + modal := base + day*25 + rec := record{ + Grade: "Non-FAQ", + Group: "Cereals", + State: "Chattisgarh", + Market: "Mock APMC " + market, + Variety: "D.B.", + District: "Balodabazar", + Commodity: "Commodity " + commodity, + PriceUnit: "Rs./Qtl", + ModalPrice: fmt.Sprintf("%d", modal), + ArrivalDate: date.Format("02-01-2006"), + } + // The last record reports no minimum or maximum, which happens in + // the real data. It must arrive as absent, not zero. + if day < *days-1 { + rec.MinPrice = fmt.Sprintf("%d", modal-100) + rec.MaxPrice = fmt.Sprintf("%d", modal+100) + } + records = append(records, rec) + } + + w.Header().Set("Content-Type", "application/json") + // A bare array, which is one of the three shapes the real API uses. + if err := json.NewEncoder(w).Encode(records); err != nil { + log.Printf("could not write the answer: %v", err) + } + }) + + log.Printf("mockagmarknet listening on %s, %d records per answer", *addr, *days) + if err := http.ListenAndServe(*addr, nil); err != nil { + log.Printf("mockagmarknet stopped: %v", err) + os.Exit(1) + } +} + +// hash makes the prices depend on what was asked for, so a wrong code is +// visible in the answer rather than silently tolerated. +func hash(s string) uint32 { + h := fnv.New32a() + _, _ = h.Write([]byte(s)) + return h.Sum32() +} diff --git a/docker-deployment/mocks/mockimd/Dockerfile b/docker-deployment/mocks/mockimd/Dockerfile new file mode 100644 index 0000000..9f73c1d --- /dev/null +++ b/docker-deployment/mocks/mockimd/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.26.1-bookworm AS builder +WORKDIR /src +COPY go.mod ./ +COPY main.go ./ +RUN CGO_ENABLED=0 go build -o mockimd . + +FROM cgr.dev/chainguard/wolfi-base:latest +WORKDIR /app +COPY --from=builder /src/mockimd . +EXPOSE 9100 +ENTRYPOINT ["./mockimd"] diff --git a/docker-deployment/mocks/mockimd/go.mod b/docker-deployment/mocks/mockimd/go.mod new file mode 100644 index 0000000..850bbd2 --- /dev/null +++ b/docker-deployment/mocks/mockimd/go.mod @@ -0,0 +1,3 @@ +module mockimd + +go 1.22.2 diff --git a/docker-deployment/mocks/mockimd/main.go b/docker-deployment/mocks/mockimd/main.go new file mode 100644 index 0000000..0b61d5f --- /dev/null +++ b/docker-deployment/mocks/mockimd/main.go @@ -0,0 +1,147 @@ +// Command mockimd stands in for IMD's Mausamgram NWP API during local +// end-to-end runs. +// +// It answers the shape the real service does -- fcstday1..N carrying date, +// rain, tmin, tmax, rhmin, rhmax, wspd and a warning -- and requires the same +// basic auth, so the adapter's credential path is exercised rather than +// skipped. Forecasts are derived from the requested point so a wrong lat/lon +// shows up as wrong numbers instead of passing silently. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "math" + "net/http" + "os" + "strconv" + "time" +) + +type forecast struct { + Date string `json:"date"` + Rain float64 `json:"rain"` + TMin float64 `json:"tmin"` + TMax float64 `json:"tmax"` + RHMin int `json:"rhmin"` + RHMax int `json:"rhmax"` + WSpd float64 `json:"wspd"` + Wind []string `json:"wind,omitempty"` + WeatherWarning string `json:"weather_warning,omitempty"` + CloudMessage string `json:"cloud_message,omitempty"` +} + +func main() { + addr := flag.String("addr", ":9100", "listen address") + user := flag.String("user", "", "basic auth username; empty with -pass means no auth") + pass := flag.String("pass", "", "basic auth password; empty with -user means no auth") + days := flag.Int("days", 3, "forecast days to return (1-5)") + flag.Parse() + + // No credential configured means none demanded. That is how this runs in the + // local stack: the registry publishes auth.scheme "none" for this upstream, + // which is what lets its baseUrl be plaintext http, and a mock that still + // demanded a password would contradict the record the adapter reads. + requireAuth := *user != "" || *pass != "" + + http.HandleFunc("/get-daily", func(w http.ResponseWriter, r *http.Request) { + if requireAuth { + gotUser, gotPass, ok := r.BasicAuth() + if !ok || gotUser != *user || gotPass != *pass { + log.Printf("401 %s %s -- basic auth missing or wrong", r.Method, r.URL.RequestURI()) + w.Header().Set("WWW-Authenticate", `Basic realm="mausamgram"`) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + } + + lat, lon, err := point(r) + if err != nil { + log.Printf("400 %s -- %v", r.URL.RequestURI(), err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // The whole query, not just the two fields this mock parses: what the + // adapter sent is decided by a mapping file, so a log that prints only + // the fields already known here cannot show a mapping change at all. + log.Printf("200 %s?%s (lat=%v lon=%v)", r.URL.Path, r.URL.RawQuery, lat, lon) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body(lat, lon, *days)) + }) + + http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "ok") + }) + + auth := "no auth" + if requireAuth { + auth = fmt.Sprintf("basic auth %s/%s", *user, *pass) + } + log.Printf("mock IMD listening on http://%s (%s, %d day forecast)", *addr, auth, *days) + if err := http.ListenAndServe(*addr, nil); err != nil { + log.Println(err) + os.Exit(1) + } +} + +// point reads the coordinates the adapter sent, which is what proves the +// request mapping produced them. +func point(r *http.Request) (float64, float64, error) { + latRaw, lonRaw := r.URL.Query().Get("lat"), r.URL.Query().Get("lon") + if latRaw == "" || lonRaw == "" { + return 0, 0, fmt.Errorf("lat and lon are required, got %q", r.URL.RawQuery) + } + lat, err := strconv.ParseFloat(latRaw, 64) + if err != nil { + return 0, 0, fmt.Errorf("lat %q is not a number", latRaw) + } + lon, err := strconv.ParseFloat(lonRaw, 64) + if err != nil { + return 0, 0, fmt.Errorf("lon %q is not a number", lonRaw) + } + return lat, lon, nil +} + +// body derives a forecast from the point, so a wrong coordinate produces wrong +// numbers rather than passing unnoticed. The last day is deliberately partial: +// a provider that reports some readings and not others is the ordinary case, +// and the mapping has to omit what was not measured. +func body(lat, lon float64, days int) map[string]any { + if days < 1 { + days = 1 + } + if days > 5 { + days = 5 + } + + out := map[string]any{"location": map[string]float64{"lat": lat, "lon": lon}} + base := math.Abs(lat) + math.Abs(lon) + + for day := 1; day <= days; day++ { + date := time.Now().AddDate(0, 0, day-1).Format("2006-01-02") + f := forecast{ + Date: date, + TMin: round(20 + math.Mod(base, 5) + float64(day)*0.4), + TMax: round(30 + math.Mod(base, 4) + float64(day)*0.3), + } + if day < days { + f.Rain = round(math.Mod(base*float64(day), 20)) + f.RHMin = 50 + day + f.RHMax = 88 + day + f.WSpd = round(3 + math.Mod(base, 3)) + f.Wind = []string{"NW", "North Westerly"} + if f.Rain > 10 { + f.WeatherWarning = "Heavy rainfall warning" + } else { + f.CloudMessage = "Partly cloudy" + } + } + out[fmt.Sprintf("fcstday%d", day)] = f + } + return out +} + +func round(v float64) float64 { return math.Round(v*10) / 10 } From d5393503f494895c1c5bd15f6d60d31c0141702a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:00:14 +0530 Subject: [PATCH 28/81] feat: seed both providers and their capability bindings [OpenAgriNet/network-adapter#4] setup.py registered the three adapter identities and left the provider's two rows to be created by hand, on the reasoning that the upstream's base URL belonged to whoever ran it. That no longer holds twice over: the upstreams are in this stack now, and the registry is not reachable from outside it, so "by hand" had no way to happen. It seeds all of it -- five participants and two capability bindings. Three node identities with the keypairs it generates, two upstreams addressed by compose service name, and a ProviderSchema row per capability carrying the method, path, timeouts and mapping URL. Both new helpers search before they write, so a re-run converges rather than failing, and keys/keys.json is reused rather than regenerated. The binding keys are the part worth being careful about. A provider step answers only when the key it was configured with matches the one built from the incoming payload, and the two are now rendered and seeded from the same .env values in the same run -- which is what keeps them from disagreeing. Changing a provider is an .env edit and a re-run. Adds the mandi capability to the provider template alongside weather, lifted from the config the oan-local stack has been tested against. Both ids are in providerSteps and both are in steps:, because a step that is declared but missing from steps: never runs and the request falls through to a 404 that does not say why. Also drops BIND_ADDR from .env.example. Nothing read it -- the loopback binds are written literally in the compose file, deliberately, since one variable that moves every port onto a public interface at once is a footgun. --- docker-deployment/.env.example | 126 +++++++---- docker-deployment/bin/setup.py | 109 ++++++++-- docker-deployment/bin/stack.sh | 8 +- .../config/adapters/provider.yaml.tmpl | 23 +- .../agmarknet/mandi-price.select.yaml | 200 ++++++++++++++++++ 5 files changed, 397 insertions(+), 69 deletions(-) create mode 100644 docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index 2f6d09c..de94fb1 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -7,15 +7,19 @@ # never written here. # ---- who can reach it ------------------------------------------------------ -# The interface the published ports bind to. 127.0.0.1 keeps the stack on the -# VM's loopback, reachable over an SSH tunnel: +# Every published port except the gateway's 80 and 443 is bound to 127.0.0.1, +# written literally in docker-compose.yml rather than taken from a variable. +# That is deliberate: one variable that moves every port to the public +# interface at once is a footgun, and the ports that should be reachable are +# reachable through the gateway instead. +# +# So from a workstation, a tunnel: # # ssh -L 9202:127.0.0.1:9202 -L 8081:127.0.0.1:8081 you@the-vm # -# Set this to 0.0.0.0 only once something in front of it is terminating TLS -# and authenticating. Behind these ports sit Keycloak's admin console and a -# registry whose write token any reader of this file can mint. -BIND_ADDR=127.0.0.1 +# Behind those loopback ports sit Keycloak's admin console and a registry whose +# write token any reader of this file can mint. Publishing one means editing +# docker-compose.yml, which is where that argument belongs. # ---- the images ------------------------------------------------------------ # Nothing is built here. `docker compose up -d` pulls these and starts them. @@ -32,6 +36,12 @@ BIND_ADDR=127.0.0.1 ADAPTER_IMAGE=ghcr.io/nisargabd/oan-adapter:latest DISCOVERY_IMAGE=ghcr.io/nisargabd/discovery-service:${TAG:-latest} +# The two mock upstreams. Their sources are in mocks/, to be built and +# published once rather than built here -- see mocks/README.md for the build +# commands and for what they deliberately get wrong. +MOCKIMD_IMAGE=ghcr.io/nisargabd/oan-mockimd:latest +MOCKAGMARKNET_IMAGE=ghcr.io/nisargabd/oan-mockagmarknet:latest + # ---- ports ----------------------------------------------------------------- REGISTRY_PORT=8081 KEYCLOAK_PORT=8080 @@ -40,6 +50,11 @@ DISCOVERY_PORT=8090 PROVIDER_ADAPTER_PORT=9200 NETWORK_ADAPTER_PORT=9201 EXP_ADAPTER_PORT=9202 +# The mocks are published on loopback for looking at them directly while +# debugging. The adapter reaches them by compose service name, not through +# these. +MOCKIMD_PORT=9100 +MOCKAGMARKNET_PORT=9101 # ---- registry -------------------------------------------------------------- # CHANGE every credential in this block before the stack is exposed. @@ -78,53 +93,72 @@ EXP_SUBSCRIBER_ID=exp.oan.dev NETWORK_SUBSCRIBER_ID=network.oan.dev PROVIDER_SUBSCRIBER_ID=provider.oan.dev -# ---- the upstream provider ------------------------------------------------- -# The provider's two registry rows are created by hand -- see README.md. -# setup.py does not create them, because the base URL belongs to whoever runs -# the upstream API. -# -# WHY THESE TWO MUST MATCH THOSE ROWS. The provider adapter decides whether a -# request is its own by building a binding key out of the incoming payload -- -# the provider id and the capability @type it carries -- and comparing that -# against the keys in its own config. setup.py renders these two values into -# that config as the one key it answers to. -# -# A disagreement fails, and how it fails depends on which side is wrong: -# -# the payload names a provider this adapter is not configured for +# ---- the two upstream providers -------------------------------------------- +# bin/setup.py creates BOTH of these in the registry, along with their +# capability bindings -- five participants and two bindings in all. It has to: +# the registry is not reachable from outside this stack, so there is no second +# way to create them. +# +# WHY THESE MUST MATCH THE ROWS. The provider adapter decides whether a request +# is its own by building a binding key out of the incoming payload -- the +# provider id and the capability @type it carries -- and comparing it against +# the keys in its own config. setup.py renders these values into that config +# AND seeds the registry from them, which is what keeps the two from +# disagreeing. Change one here and re-run setup.py; changing only one side is +# how the failures below happen. +# +# A disagreement fails, and how depends on which side is wrong: +# +# the payload names a provider no step is configured for # 404 NET_ENTITY_NOT_FOUND, "this module serves no capability matching the -# request". The step passes it through -- which is what lets one adapter -# host several capabilities -- and nothing behind it answers. +# request". Each step passes through what is not its own -- which is what +# lets one adapter host both capabilities -- and nothing behind them +# answers. # -# the adapter IS configured for the key but the registry has no matching +# a step IS configured for the key but the registry has no matching # ProviderSchema row -# 502 with an EMPTY body. The reason, "no call plan for ", appears -# only in the provider adapter's log: -# docker compose logs provider-adapter | grep "no call plan" -# -# So these two must equal participantId and capabilityCode in the -# ProviderSchema row, exactly. While the adapter carries ONE configured key, -# onboarding a second provider is an edit here, a re-run of bin/setup.py and a -# restart -- a registry entry alone is not enough. -PROVIDER_PARTICIPANT_ID=my-weather-api +# 404 too, now naming the binding with no active record. Before that it was +# a 500 with the reason only in the log. +# +# The base URLs are compose service names: both upstreams are mocks called from +# inside this network and reached from nowhere else. Pointing a capability at a +# real API means a new Participant and ProviderSchema row, written from inside +# the stack, and the id here updated to match. +PROVIDER_PARTICIPANT_ID=mausamgram-mock PROVIDER_CAPABILITY=openagrinet:WeatherObservation +MAUSAMGRAM_BASE_URL=http://mockimd:9100 +MAUSAMGRAM_PATH=/get-daily -# The mapping the provider adapter fetches, request and response in one file. -# The registry row holds the full URL and the adapter fetches it verbatim. -# -# This repo's own copy, served over the raw CDN -- the same file that sits in -# config/mappings/ beside these configs, so what the adapter fetches and what -# a reader reviews are one file and cannot drift. +MANDI_PARTICIPANT_ID=agmarknet-mock +MANDI_CAPABILITY=openagrinet:MandiPrice +MANDI_BASE_URL=http://mockagmarknet:9101 +MANDI_PATH=/v1/fetch-agmarknet-vistaar + +# Agmarknet's Vistaar API takes its token as a QUERY parameter, which is why +# the adapter needs authScheme query for it. The value lives here and reaches +# the adapter as an environment variable; it is never in a config file or in +# the registry. The mock answers 401 without it, which is what proves the +# adapter sent one. +MANDI_TOKEN=local-mandi-token + +# How many records each mock answers with. The mappings read however many +# arrive, so these are the knobs for checking that they do. mockagmarknet's +# last record reports no minimum or maximum, as the real data sometimes does. +MOCKIMD_DAYS=3 +MOCKAGMARKNET_DAYS=2 + +# The mappings each provider adapter fetches, request and response in one file +# per binding-action. The registry row holds the full URL and the adapter +# fetches it verbatim. # -# It is a URL and not a path because the registry publishes the full URL and -# the adapter fetches it verbatim. A mapping therefore has to be reachable -# before it can be tested, which means what this stack exercises is exactly -# what any consumer fetches. +# This repo's own copies, served over the raw CDN -- the same files that sit in +# config/mappings/ beside these configs, so what the adapter fetches and what a +# reader reviews are one file and cannot drift. # -# Note the branch in the path. Once this merges, change it to the default +# Note the branch in the paths. Once this merges, change it to the default # branch, or pin a tag so a deployment is not following a moving file. # -# To change the mapping: edit config/mappings/, push, and the next cache -# expiry picks it up -- or publish a fork anywhere that serves raw text over -# https and put that URL in the ProviderSchema row instead. +# To change a mapping: edit config/mappings/, push, and the next cache expiry +# picks it up -- about a minute for the adapter plus a few for GitHub's CDN. MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml +MANDI_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 4af3038..5f53e19 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -213,6 +213,43 @@ def node(participant_id, name, role, public_key): "role": role, "keys": signing_key_block(public_key)} +def upstream(participant_id, name, base_url): + """An ordinary API the provider adapter calls. + + No role and no keys: it has never heard of Beckn, so it signs nothing and + nothing verifies it. Both are permitted by the schema but neither is read -- + a signature is checked against the node identity that signed it. + + No credential either. The adapter presents one from its own config, naming + the environment variable it comes from, so nothing secret is held here.""" + return {"participantId": participant_id, "name": name, "type": "upstream", + "status": "active", "baseUrl": base_url} + + +def ensure_binding(bearer, participant_id, capability, path, mapping_url): + """Create a capability binding only when absent. + + actions is a list, not a map: the registry treats every nested object as an + entity and injects an osid into it, which a map cannot carry. It is also + what lets one action be retired without touching the others. + + mappings is one reference carrying both directions, because the response + mapping reads what the request mapping resolved.""" + binding = f"{participant_id}|{capability}" + if search("ProviderSchema", {"bindingKey": {"eq": binding}}): + print(f" {binding}: already present") + return + result = post("ProviderSchema", { + "bindingKey": binding, "participantId": participant_id, + "capabilityCode": capability, "status": "active", + "actions": [{"action": "select", "method": "GET", "path": path, + "mappings": mapping_url, + "timeoutMs": 15000, "retryMax": 2, + "status": "active"}]}, bearer) + print(f" {binding}: {result['params']['status']} " + f"{result['params'].get('errmsg', '')[:160]}") + + def ensure_participant(bearer, participant_id, payload): """Create only when absent. Delete here is soft and keeps the unique index, so a recreate would fail on a duplicate key rather than replacing.""" @@ -227,8 +264,14 @@ def ensure_participant(bearer, participant_id, payload): def seed(identities): wait_for_registry() bearer = token() - print("registry: the three adapter identities") + # Five participants and two capability bindings, all of it from here. + # + # The registry is not reachable from outside this stack -- no published + # port beyond loopback and no proxy host in front of it -- so there is no + # second way to create these. Everything the network needs to answer a + # request has to exist by the time this returns. + print("registry: three adapter identities") for role, name, network_role in ( ("exp", "OAN experience layer adapter", "consumer"), ("network", "OAN network layer adapter", "network"), @@ -238,6 +281,29 @@ def seed(identities): node(identity["participantId"], name, network_role, identity["signingPublic"])) + # The two upstreams, addressed by compose service name: they are called from + # inside this network and nowhere else. + print("registry: two upstream providers") + weather = env("PROVIDER_PARTICIPANT_ID") + ensure_participant(bearer, weather, + upstream(weather, "IMD Mausamgram NWP (mock)", + env("MAUSAMGRAM_BASE_URL", "http://mockimd:9100"))) + mandi = env("MANDI_PARTICIPANT_ID") + ensure_participant(bearer, mandi, + upstream(mandi, "Agmarknet Vistaar (mock)", + env("MANDI_BASE_URL", "http://mockagmarknet:9101"))) + + # And what each of them answers. The binding key is participantId piped to + # capabilityCode, and it has to match what the provider adapter was + # rendered with -- both come from the same .env, which is what keeps them + # from disagreeing. + print("registry: two capability bindings") + ensure_binding(bearer, weather, env("PROVIDER_CAPABILITY"), + env("MAUSAMGRAM_PATH", "/get-daily"), env("MAPPING_URL")) + ensure_binding(bearer, mandi, env("MANDI_CAPABILITY"), + env("MANDI_PATH", "/v1/fetch-agmarknet-vistaar"), + env("MANDI_MAPPING_URL")) + def key_osids(identities): """Read back each key's osid, and check the registry still holds the public @@ -277,6 +343,7 @@ def key_osids(identities): def render(identities): print("configs:") binding = f"{env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')}" + mandi_binding = f"{env('MANDI_PARTICIPANT_ID')}|{env('MANDI_CAPABILITY')}" for role in ("exp", "network", "provider"): identity = identities[role] template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() @@ -288,7 +355,8 @@ def render(identities): (f"__{prefix}_SIGNING_PUBLIC__", identity["signingPublic"]), (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"]), - ("__PROVIDER_BINDING_KEY__", binding)): + ("__PROVIDER_BINDING_KEY__", binding), + ("__MANDI_BINDING_KEY__", mandi_binding)): template = template.replace(placeholder, value) if "__" in template: sys.exit(f"setup: {role}.yaml still has unrendered placeholders") @@ -318,20 +386,23 @@ def render(identities): KEYS.write_text(json.dumps(identities, indent=2)) render(identities) print(f""" -ready -- the adapters can now sign and verify each other. - -Still to do by hand, because the base URL is not this stack's to know: - - 1. give the upstream API a URL this VM can reach. Tunnelled from a laptop, - that is: ngrok http 9100 - 2. register it -- two rows, see README.md: - Participant type "upstream", baseUrl = that https URL - ProviderSchema bindingKey {env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')} - 3. docker compose up -d - -The bindingKey above is what the provider adapter was just configured to -answer to, and the ProviderSchema row has to match it exactly. A payload -naming anything else is answered 404 "this module serves no capability -matching the request". A row that is missing, while the adapter is configured -for the key, is answered 502 with an empty body and explained only in -`docker compose logs provider-adapter`.""") +ready. The registry holds five participants and two capability bindings, and +the adapter configs are rendered, so nothing further has to be created by hand. + + {env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')} + {env('MANDI_PARTICIPANT_ID')}|{env('MANDI_CAPABILITY')} + +Those are the binding keys the provider adapter answers to. They were rendered +into its config from the same .env this seeded the registry from, which is what +keeps the two from disagreeing -- and a disagreement is quiet: a payload naming +anything else is answered 404 "this module serves no capability matching the +request". + +Both providers are mocks reached by compose service name. Pointing a capability +at a real upstream is a registry write, not a change here: a new Participant +with its base URL, a ProviderSchema row naming it, and the adapter's +{env('PROVIDER_CAPABILITY')} or {env('MANDI_CAPABILITY')} entry in .env +updated to match. The registry is not reachable from outside this stack, so +that write happens from here. + +Next: make up, then import postman-collection/ and run it.""") diff --git a/docker-deployment/bin/stack.sh b/docker-deployment/bin/stack.sh index 65a0509..5f3e63d 100755 --- a/docker-deployment/bin/stack.sh +++ b/docker-deployment/bin/stack.sh @@ -129,13 +129,17 @@ up_registry_tier() { # beside them. Safe to re-run: keys come from keys/keys.json once it exists, # and participants already registered are left alone. up_setup() { - step 2 "$1" "bin/setup.py -- keys, adapter identities, adapter configs" + step 2 "$1" "bin/setup.py -- keys, five registry participants, adapter configs" python3 bin/setup.py } # Only now do the bind-mounted config files exist. up_adapters() { - step 3 "$1" "adapters (provider, network, exp)" + # The mocks are named here rather than left to provider-adapter's + # depends_on, so a failure to pull one is reported as its own step instead + # of as an adapter that will not start. + step 3 "$1" "mock upstreams and adapters (provider, network, exp)" + docker compose up -d mockimd mockagmarknet docker compose up -d provider-adapter network-adapter exp-adapter } diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 5d90e4a..8ffcf06 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -108,11 +108,30 @@ modules: config: bindingKeys: "__PROVIDER_BINDING_KEY__" authScheme: none - + # A second capability in the same pipeline, from a different domain + # package. Each step recognises its own binding key and passes + # through anything else, so adding one is an entry here rather than a + # change to a routing table. + # + # Agmarknet takes its token as a QUERY parameter. The adapter holds + # the parameter's name and the name of the variable carrying the + # value, never the value -- and it redacts it from the URL it logs. + - id: mandi + config: + bindingKeys: "__MANDI_BINDING_KEY__" + authScheme: query + queryName: token + queryValueEnv: MANDI_TOKEN + + # Declaring a provider step above is not enough: this list is what runs. + # Both provider steps sit in the pipeline in order, and each recognises + # its own binding key and passes through anything else -- so which one + # answers is decided by the payload, not by this order. steps: - validateSign # the sender's key, from the registry - validateSchema # the pinned Beckn v2 spec - - weather # resolve, map out, call, map back + - weather # openagrinet:WeatherObservation, or pass through + - mandi # openagrinet:MandiPrice, or pass through - signAck # signs whatever the step answered with # The outbound leg: the provider's own catalogue system publishing to the diff --git a/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml b/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml new file mode 100644 index 0000000..2f0e70b --- /dev/null +++ b/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml @@ -0,0 +1,200 @@ +# Agmarknet Vistaar, openagrinet:MandiPrice, select. Both directions, one file. +# +# One file per binding-action rather than one per direction, because both legs of +# an exchange are one contract: the response has to answer the request that was +# sent, and splitting them lets one change without the other. +# +# The upstream is Agmarknet's Vistaar select. It takes governed codes -- state, +# district, market, commodity -- plus a date range, and every one of them is in +# the payload, so nothing here needs resolving before the call. That is why the +# mandi plugin has no prerequisites: a MandiPrice select names the market it +# wants rather than a point to search from. +# +# NOTHING HERE IS OUTSIDE THE PACK. openagrinet:MandiPrice v0.1 carries every +# field this answer sets. Where the upstream reports something the pack has no +# home for, it is dropped rather than invented. + +# What this capability cannot serve, refused before the provider is called. +# +# The pack requires none of these: a MandiPrice select is OnDemand, and that +# branch requires only supportedCommodities and supportedPriceFields. It leaves +# market and validity optional, and defines market.district and market.state as +# "name or governed code". So a payload can be perfectly valid and still be +# unanswerable by this upstream, which wants codes and a date range. +# +# Refusing here names what is missing. Sending it anyway earns a 400 from +# Agmarknet, or worse an empty result that reads as "no prices". +required: + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.supportedCommodities[0].code) + ) + message: "this capability needs a commodity code in supportedCommodities[0].code" + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.market.state) and $exists($ra.market.district) + ) + message: "this capability needs governed state and district codes in market" + - check: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + $exists($ra.validity.startsAt) and $exists($ra.validity.endsAt) + ) + message: "this capability needs a validity window; it reports prices over a date range" + +# The upstream is a GET, so this object becomes the query string. The token is +# not here and must never be: it comes from the adapter's authScheme query, +# whose value is read from an environment variable. This file is published. +# +# marketcode is sent when the payload names one and omitted otherwise, which is +# what the upstream expects: without it the query widens from one market to the +# whole district. +request: | + ( + $ra := beckn.message.contract.commitments[0].resources[0].resourceAttributes; + + /* The upstream wants dd-MM-yyyy; the pack's validity is an ISO date. A + substring reorder rather than a date library, because these are dates + with no time and no zone, and $fromMillis would invent both. */ + $ddmmyyyy := function($iso) { + $substring($iso, 8, 2) & "-" & $substring($iso, 5, 2) & "-" & $substring($iso, 0, 4) + }; + + $base := { + "statecode": $ra.market.state, + "districtcode": $ra.market.district, + "commoditycode": $ra.supportedCommodities[0].code, + "from_date": $ddmmyyyy($ra.validity.startsAt), + "to_date": $ddmmyyyy($ra.validity.endsAt) + }; + + $exists($ra.market.marketCode) + ? $merge([$base, {"marketcode": $ra.market.marketCode}]) + : $base + ) + +# One resource per price record, in Direct mode. +# +# Direct is what the pack requires of an answer: the resource now CARRIES the +# observation rather than advertising that it could obtain one. It requires +# source, commodity, market, arrivalDate, prices and generatedAt, and all six +# are set below. +# +# The upstream's records use Title Case keys WITH SPACES -- `Modal Price`, not +# modalPrice -- so they need backticks, and its prices are STRINGS, so they need +# $number() before they satisfy the pack's numeric types. +response: | + ( + $records := $type(response) = "array" ? response + : $exists(response.data) ? response.data + : $exists(response.records) ? response.records + : []; + + $selected := beckn.message.contract.commitments[0]; + $ra := $selected.resources[0].resourceAttributes; + + /* Bound once because it is used twice -- for a resource's own id and for + the offer's reference to it. Two copies of one expression is how a + dangling reference gets reintroduced. */ + /* Built from CODES, not the names the upstream reports. A market name + carries spaces and a commodity name carries brackets -- "Kasdol APMC", + "Paddy(Common)" -- and an identifier that a consumer may put in a URL or + a filter should not. The codes are already in the payload, so they cost + nothing, and they are stable where a display name is not. + + The market code is optional: without it the query widened to the whole + district, so the district code is what identifies the scope. */ + $iso := function($ddmmyyyy) { + $substring($ddmmyyyy, 6, 4) & "-" & $substring($ddmmyyyy, 3, 2) & "-" & $substring($ddmmyyyy, 0, 2) + }; + + $scope := $exists($ra.market.marketCode) ? $ra.market.marketCode : $ra.market.district; + $resourceId := function($r) { + "res:agmarknet:" & $scope & ":" & $ra.supportedCommodities[0].code + & ":" & $iso($r.`Arrival Date`) + }; + + /* dd-MM-yyyy back to ISO, so the answer speaks the pack's date format + rather than the upstream's. */ + + /* Absent rather than present-and-empty: a consumer must be able to tell + "the market reported no minimum" from "the minimum was zero". */ + $priced := function($value) { $exists($value) ? $number($value) }; + + { + "context": { + "version": beckn.context.version, + "action": "on_select", + "networkId": beckn.context.networkId, + "transactionId": beckn.context.transactionId, + "messageId": beckn.context.messageId, + "timestamp": $now() + }, + "message": { + "contract": { + "commitments": [ + { + "status": { + "descriptor": { "code": "DRAFT", "name": "Draft" } + }, + /* The offer is echoed, but its references are not: the request + named an abstract price enquiry and the answer returns the + concrete observations. Leaving resourceIds as they arrived + would point the offer at an id appearing nowhere here. */ + "offer": $merge([ + $selected.offer, + { "resourceIds": [$map($records, function($r) { $resourceId($r) })] } + ]), + /* Wrapped: JSONata collapses a one-element sequence to a bare + value, so a single-record answer would return an object where + every other count returns a list. */ + "resources": [$map($records, function($r) { + { + "id": $resourceId($r), + /* Required by Commitment.resources in the Beckn v2 spec, + which defines no quantity property and carries no Quantity + schema at all -- a defect upstream. One resource is one + market's observation for one day, so one. */ + "quantity": 1, + "resourceAttributes": { + "@context": "https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld", + "@type": "openagrinet:MandiPrice", + "informationMode": "Direct", + "subjectCategories": $ra.subjectCategories, + "source": { + "sourceId": "agmarknet", + "sourceName": "Agmarknet Vistaar" + }, + "commodity": { + "code": $ra.supportedCommodities[0].code, + "name": $r.Commodity + }, + "commodityGroup": $r.Group, + "variety": $r.Variety, + "grade": $r.Grade, + "market": { + "marketName": $r.Market, + "marketCode": $ra.market.marketCode, + "district": $r.District, + "state": $r.State + }, + "arrivalDate": $iso($r.`Arrival Date`), + "prices": { + "minimum": $priced($r.`Min Price`), + "maximum": $priced($r.`Max Price`), + "modal": $number($r.`Modal Price`), + "currency": "INR", + "unit": $r.`Price Unit` + }, + "generatedAt": $now() + } + } + })] + } + ] + } + } + } + ) From 95738c98a11a2c117f94d89791ae0ec2ec8d74ea Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:00:27 +0530 Subject: [PATCH 29/81] refactor: keep the registry off the public edge [OpenAgriNet/network-adapter#4] The registry was on oan-edge so NPM could resolve it, with a documented route, an allowlist to paste into the host's Advanced tab and an Access List to attach. The decision is the other way now: it is on oan-internal only, so a proxy host pointed at it fails to resolve rather than quietly working. The argument against the route is that it cannot be reduced to reads. SunbirdRC uses POST for both -- /Participant/search reads, /Participant creates -- so no method rule separates them and a host forwards the whole API. What kept writes out was never the route: it was that nothing outside the VM can mint a Keycloak token, because Keycloak publishes on 127.0.0.1. That is a decision made elsewhere in this file, and a registry route would have depended on it silently -- publish Keycloak later for an unrelated reason and the write surface opens with it, with nothing in the route changing to say so. So the allowlist goes with it. It was a real defence, but a defence for a host that should not exist is worse than no host: it makes the route look considered. Reaching the registry is now an SSH tunnel, and everything that has to write to it -- which is all of setup.py -- runs on the VM. --- .../config/gateway/npm-advanced/registry.conf | 47 ------------------- docker-deployment/docker-compose.yml | 37 ++++++++------- 2 files changed, 19 insertions(+), 65 deletions(-) delete mode 100644 docker-deployment/config/gateway/npm-advanced/registry.conf diff --git a/docker-deployment/config/gateway/npm-advanced/registry.conf b/docker-deployment/config/gateway/npm-advanced/registry.conf deleted file mode 100644 index 936f3c0..0000000 --- a/docker-deployment/config/gateway/npm-advanced/registry.conf +++ /dev/null @@ -1,47 +0,0 @@ -# NOT loaded automatically. Paste into the registry proxy host's Advanced tab -# ("Custom Nginx Configuration"), then Save. -# -# It reduces a host that would otherwise forward the whole registry API to the -# two endpoints a network peer actually needs, and 403s everything else. -# -# ---------------------------------------------------------- why an allowlist -# -# Because a denylist here fails open. SunbirdRC uses POST for both search and -# create -- POST /api/v1/Participant/search reads, POST /api/v1/Participant -# writes -- so no method rule separates them, and a list of paths to block is -# a list someone has to keep complete forever. -# -# NPM gives no clean way to say "only these paths". Its generated `location /` -# catches everything, and a second `location /` in this box is a duplicate -# that nginx refuses to start on. Exact-match `location =` blocks can only -# subtract, which is the denylist again. -# -# So the decision happens in the rewrite phase, before location matching, with -# the one `if` construction that is documented as safe: `if` containing -# nothing but `return`. Default deny, then name what is allowed. -# -# --------------------------------------------------------------- what is not -# -# This is a path filter, not authentication. It does not stop anyone who -# reaches it from reading the full participant list -- public keys, baseUrls, -# who is on this network -- because that is precisely what search returns. -# Pair it with an NPM Access List (Satisfy Any OFF) if that list is not -# something you would publish. - -set $registry_allowed 0; - -# The Beckn-facing reads. Both are POST with a filter body; the (\?|$) anchor -# keeps /searchsomething from matching the prefix. -if ($request_uri ~ "^/api/v1/(Participant|ProviderSchema)/search(\?|$)") { - set $registry_allowed 1; -} - -# Belt and braces: search is POST, and anything else arriving at that path is -# not the call this host exists to serve. -if ($request_method != POST) { - set $registry_allowed 0; -} - -if ($registry_allowed = 0) { - return 403; -} diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 8cb3d3d..1f89a58 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -160,27 +160,28 @@ services: image: ghcr.io/sunbird-rc/sunbird-rc-core:${REGISTRY_VERSION} container_name: oan-registry restart: unless-stopped - # On oan-edge so NPM can resolve it, which is what makes a public proxy - # host for the registry possible at all. Read this before relying on it: + # oan-internal ONLY, and that is the decision, not an omission. NPM sits + # on oan-edge alone, so it cannot resolve the name `registry` -- a proxy + # host pointed here fails to start rather than quietly working. Attaching + # oan-edge is the one edit that would make the registry publishable, so it + # is the one edit to refuse. # - # POST /api/v1/Participant/search takes NO token -- that is the call a - # peer actually needs, and publishing it is defensible. Everything else - # under /api/v1/ is a write, and a write needs a Keycloak token. Those are - # unobtainable from outside today for one reason only: Keycloak publishes - # on 127.0.0.1. So the safety of this route rests on a decision made - # forty lines up, not on anything the route itself does -- publish - # Keycloak later and the write surface opens with it, silently. + # Why not route it. POST /api/v1/Participant/search takes NO token, which + # makes it tempting: it is the one call a network peer needs. But + # SunbirdRC uses POST for both reads and writes -- /Participant/search + # reads, /Participant creates -- so no method rule separates them, and a + # route publishes the write surface the moment anything can mint a token. + # Today nothing can, for one reason only: Keycloak publishes on + # 127.0.0.1. That is a decision forty lines up, and a route here would + # silently depend on it. # - # Which is why this host is not meant to be served bare. Two things go - # with it, both in README under "Adding a route": - # - an NPM Access List with Satisfy Any OFF, and - # - config/gateway/npm-advanced/registry.conf, which reduces the host to - # the two search endpoints and 403s the rest. + # What this costs: nothing outside the VM can write a registry row, which + # is why bin/setup.py seeds all five participants and both bindings from + # inside. It is also why the Postman collection has no registry request -- + # there is no way for one to work. # - # If neither is in place, take oan-edge back off rather than leaving it - # attached "for later" -- an attached service is one form field away from - # being public. - networks: [oan-internal, oan-edge] + # Reaching it for debugging is an SSH tunnel to ${REGISTRY_PORT}. + networks: [oan-internal] volumes: # Schemas are read at startup, so a change here needs this service # restarted before the registry will honour it. From 5ad3d6df5e6d927c916d0f0437101d54d531dd3d Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:00:42 +0530 Subject: [PATCH 30/81] docs: rewrite the flow around make up, and drop the registry requests [OpenAgriNet/network-adapter#4] The README described a flow that no longer exists: compose up, then create the provider's rows by hand, with an ngrok URL for an API you brought yourself. It is `cp .env.example .env && make up` now, and nothing has to be created afterwards. "Register the provider" becomes "What is in the registry, and why you did not create it" -- what the five rows are, what a binding key is for, and how to repoint a capability at a real API. Routing the registry is replaced by why it is the one service not to route, with discovery as the worked example instead. The troubleshooting section leads with the two binding-key failures and which 404 means which side is wrong. The Postman collection loses all five registry requests. Not a trim: the registry has no route, so none of them could work from outside the VM, and setup.py does what they did. Six requests remain -- publish, discover and select for each capability -- 32 assertions, and nothing left to fill in, since the two provider ids are prefilled with what the stack deploys. The mandi select's networkId was still oan-local's value; it is a variable now. Two corrections while checking claims rather than copying them: - The old text said publish is not validated because the spec does not define it. The spec does define /catalog/publish, the validator indexes it under the action these payloads send, and both publish bodies validate against it with no errors. It is off because validateSchema is absent from those two modules' steps: -- a choice pending a test, not a limitation. - A binding key with no matching ProviderSchema row was documented as a 502 with an empty body. It has been a 404 naming the binding since the adapter fix; the 502 text predated it. --- docker-deployment/README.md | 577 +++++++++++------- .../OAN-dev-flow.postman_collection.json | 386 +++++------- .../postman-collection/README.md | 93 ++- 3 files changed, 549 insertions(+), 507 deletions(-) diff --git a/docker-deployment/README.md b/docker-deployment/README.md index 30589b4..fab93e9 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -1,7 +1,9 @@ # OAN stack, on Docker Compose The whole OpenAgriNet stack for a **dev deployment on a VM**: the registry, the -discovery service, and the three adapters. One compose file, one config folder. +discovery service, the three adapters, and a mock upstream per capability so a +request has something to answer it. One compose file, one config folder, +`make up`. This is a dev environment. It is not production: the adapter signing keys sit in a config file on disk, nothing terminates TLS, and every credential shipped @@ -13,9 +15,17 @@ Running here: - **registry** — SunbirdRC, plus its Postgres and Keycloak. Holds who is on the network, their public keys, and which upstream API answers which capability. + Published on the VM's loopback only, and deliberately given no route through + the gateway — see below. - **discovery** — catalogue search, plus its own Postgres. - **three adapters** — experience, network and provider. Same image, three configs. +- **two mock upstreams** — one standing in for Mausamgram's forecast API, one + for Agmarknet's Vistaar prices. Sources in `mocks/`; they are pulled as + published images like everything else. They exist so the stack answers a + select end to end out of the box, with no external API and no ngrok tunnel. + Loopback only, and the adapter reaches them by compose service name rather + than through the published port. - **gateway** — Nginx Proxy Manager, the only container that publishes on a routable interface. Routes to the three adapters, and issues and renews the Let's Encrypt certificates from its own UI. Profile `gateway`. @@ -27,13 +37,17 @@ stack, and HyperDX is the heaviest thing here. Deliberately **not** here: -- **the provider API.** Whoever is testing runs it themselves and gives it a - URL the VM can reach. The provider adapter never has that address in a config - file — it reads it from the registry per request, so repointing it is a - registry write and nothing more. -- **the provider's registry rows.** Those two are created by hand, because the - base URL belongs to whoever runs the API. `bin/setup.py` registers only the - three adapters. +- **a route to the registry.** It is reachable from inside the compose network + and over an SSH tunnel to the VM, and from nowhere else. Nothing in front of + it authenticates, and SunbirdRC uses POST for both reads and writes, so a + route would expose creates as readily as searches. This is why `bin/setup.py` + seeds everything: with no public registry there is no second way to write a + row, and a Postman request could not do it. +- **a real provider API.** The mocks answer the same shapes. Pointing a + capability at something real is a registry write — a new Participant and + ProviderSchema row, made from inside the stack — and a base URL in `.env`. + The provider adapter never holds that address in a config file; it reads it + from the registry per request. ## Reaching it @@ -134,7 +148,6 @@ Worth being exact about, because the two look alike in the repo: | `config/gateway/npm-custom/http_top.conf` | **Automatic.** NPM includes it at the top of its `http` block. Declares the `exp` rate-limit zone and `limit_req_status 429`. | | `config/gateway/npm-custom/server_proxy.conf` | **Automatic.** Included in every proxy host's server block. Holds the `/publish` deny. | | `config/gateway/npm-advanced/exp.conf` | **Manual.** Paste into the experience host's Advanced tab. Applies `limit_req` to that host only, since a 10 r/s ceiling on signed peer traffic would throttle for no security gain. | -| `config/gateway/npm-advanced/registry.conf` | **Manual.** Paste into the registry host's Advanced tab, if you create one. Reduces the host to the two search endpoints and 403s the rest. | The manual one is in a file anyway because NPM's Advanced field is a textarea in a database row: nothing diffs it and nothing reviews it. Keeping the source @@ -181,86 +194,93 @@ the host → Custom Locations → add e.g. `/v2` forwarding to another service. That keeps one certificate and one DNS record, at the cost of NPM's generated config growing a location block you cannot see in the UI's main view. -#### Worked example: the registry +#### Worked example: discovery -`registry` is already on `oan-edge` in `docker-compose.yml`, so step 1 is -done — but read the comment there before you use it, because the mechanics are -the easy part. +`discovery` is on `oan-internal` only, so this is the two-step case — the one +where the network split does the work. -**What is actually reachable once you route it.** `POST /api/v1/Participant/search` -takes no token; that is the call a peer needs, and publishing it is defensible. -Everything else under `/api/v1/` is a write, and writes need a Keycloak token. -Those are unobtainable from outside **today for one reason only**: Keycloak -publishes on `127.0.0.1`. The safety of this route therefore rests on a -decision made elsewhere in the compose file. Publish Keycloak later and the -registry's write surface opens along with it, with nothing on this host -changing to say so. +**Step 1, attach it to `oan-edge`** in `docker-compose.yml`: -There is a second wrinkle even for someone who has a token. The registry -validates a token's issuer against `http://keycloak:8080/auth/realms/…`, the -**container-internal** address, which is why the token request further down -this README carries `X-Forwarded-Host: keycloak:8080`. A token minted through -any other hostname is rejected with a 401 and an empty body. So "it 401s -through the proxy but works over the tunnel" is expected, not a proxy bug. +```yaml + discovery: + networks: [oan-internal, oan-edge] +``` + +then `docker compose up -d discovery`. Until this, NPM cannot resolve the name +`discovery` at all and a host pointed at it fails DNS rather than working. -**Create the host.** Hosts → Proxy Hosts → Add: +**Step 2, create the host.** Hosts → Proxy Hosts → Add: | Field | Value | |---|---| -| Domain | `registry.oan.example.com` | +| Domain | `discovery.oan.example.com` | | Scheme | `http` | -| Forward Hostname | `registry` — the compose service name, not `oan-registry` | -| Forward Port | `8081` — the **container** port. Not `REGISTRY_PORT`, which is only what loopback publishes it as | +| Forward Hostname | `discovery` — the compose service name, not `oan-discovery` | +| Forward Port | `8080` — the **container** port. Not `DISCOVERY_PORT`, which is only what loopback publishes it as | | Block Common Exploits | on | -**Then both guards, before you point anything at it.** +**Step 3, put an Access List on it,** because discovery answers +unauthenticated and `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` in this +build. Nothing behind the edge will refuse a caller, so the edge is the only +authentication there is. -1. Advanced tab → paste `config/gateway/npm-advanced/registry.conf`. That - reduces the host to `Participant/search` and `ProviderSchema/search` and - 403s everything else. It is an allowlist rather than a list of things to - block, because SunbirdRC uses POST for both search and create — no method - rule separates a read from a write, so a denylist is a list someone has to - keep complete forever. - -2. Access Lists → Add, then assign it on the host's Details tab. **Satisfy Any - off**, so an address *and* a password are needed. The path filter is not - authentication: search returns the full participant list — public keys, - baseUrls, who is on this network — to anyone who reaches it. - -**Check it does what you think:** +**Check it:** ```sh -curl -s -o /dev/null -w '%{http_code}\n' -X POST \ - https://registry.oan.example.com/api/v1/Participant/search \ - -u user:pass -H 'Content-Type: application/json' -d '{"filters":{}}' # 200 - -curl -s -o /dev/null -w '%{http_code}\n' -X POST \ - https://registry.oan.example.com/api/v1/Participant \ - -u user:pass -H 'Content-Type: application/json' -d '{}' # 403 +curl -s -o /dev/null -w '%{http_code}\n' \ + https://discovery.oan.example.com/health -u user:pass # 200 curl -s -o /dev/null -w '%{http_code}\n' \ - https://registry.oan.example.com/api/v1/Participant/search # 401 + https://discovery.oan.example.com/health # 401 ``` -403 on the second is the Advanced paste; 401 on the third is the Access List. -If either returns 200, one of the two guards is not attached — and the failure -is silent, so this is worth re-running after any NPM change. +If the second returns 200 the Access List is not attached, and that failure is +silent — worth re-running after any NPM change. + +#### The registry is the one you do not route + +It will look like the obvious candidate: `POST /api/v1/Participant/search` +takes no token, and it is exactly the call a network peer needs. Route it +anyway and you have published more than that. + +SunbirdRC uses POST for **both** reads and writes — `/Participant/search` +reads, `/Participant` creates — so no method rule tells one from the other. A +proxy host forwards the whole API. What keeps writes out today is not the +route: it is that nothing outside the VM can mint a Keycloak token, because +Keycloak publishes on `127.0.0.1`. That is a decision made elsewhere in the +compose file, and a registry route would depend on it silently. Publish +Keycloak later for an unrelated reason and the registry's write surface opens +with it, with nothing in the route changing to say so. + +So `registry` is on `oan-internal` only and stays there. NPM cannot resolve +the name, which means the refusal is structural rather than a proxy host +somebody remembered not to create. + +Two consequences worth knowing, because both look like bugs otherwise: + +- **`bin/setup.py` has to seed everything** — all five participants and both + capability bindings — since there is no other way to write a row. It runs on + the VM against `127.0.0.1`. +- **the Postman collection has no registry request.** Not an omission; one + could not work. -If you decide against the route, take `oan-edge` back off `registry` in -`docker-compose.yml` rather than only deleting the proxy host. An attached -service is one form field away from being public. +Reaching it to look at a row is an SSH tunnel, covered further down. + +If a peer genuinely needs to read participants from outside, the answer is a +route to something that serves only that read — not a route to the registry. #### Before you route the ones already here -Three of the internal services will look like obvious candidates. They are -not equivalent: +Several internal services will look like obvious candidates. They are not +equivalent: | | What routing it publishes | |---|---| -| **discovery** | Read-mostly catalogue search. The most defensible of the three, and still: it answers unauthenticated, and `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` with nothing behind it in this build. Put an Access List on it. | -| **registry** | The network's identity records. Reads are unauthenticated, writes need a Keycloak token. Publishable, but only cut down to the search endpoints and behind an Access List — see below. | +| **discovery** | Read-mostly catalogue search. The most defensible of these, and still: it answers unauthenticated, and `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` with nothing behind it in this build. Put an Access List on it. | +| **registry** | No — see above. A proxy host forwards reads and writes alike, and it is off `oan-edge` so one cannot be created. | | **keycloak** | An admin console with a realm imported from a file that ships `no-user` / `no-user-password` and an admin-api client secret. Do not publish it. | | **hyperdx** | `clickstack-local` runs single-user with **no login at all**. Publishing it hands over every trace and log the stack has collected. If it must be shared, switch to `clickstack-all-in-one` and set up a team first. | +| **the two mocks** | Pointless and confusing: they exist to be called from inside by the provider adapter, and they invent their data. Nothing outside has a reason to reach them. | | **registry-db, discovery-db** | No. Use `docker compose exec`, or a tunnel. | The pattern: publishing a service that has no authentication of its own means @@ -340,61 +360,70 @@ On the VM: - 16 GB of RAM if you run the `observability` profile — ClickHouse alone wants 2-4 GB on top of the two JVM services. 8 GB is workable without it. -And a URL the VM can reach for the upstream provider API. If that API runs on -someone's laptop, [ngrok](https://ngrok.com/) or any equivalent tunnel gives it -one. +Nothing else. No external API and no tunnel: the two mock upstreams are part +of the stack, so a select has something to answer it the moment it comes up. + +`bin/bootstrap-ubuntu.sh` installs the first two on a fresh Ubuntu VM. ## Bring it up ```sh cp .env.example .env +make up ``` -Read `.env` before going on. Four things in it matter: +Read `.env` first. Two things in it matter before a first run: -- `TAG` — pins the discovery service. Unset means `latest`; set it to deploy a - known build instead of whatever `latest` points at today: - `TAG=v0.3.1 docker compose up -d`. The images themselves are already named in - `.env.example` and are pulled, never built. -- **the credentials.** All shipped defaults. Change them. -- `BIND_ADDR` — see above. -- `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` have to match the registry - rows created further down. +- **the credentials.** All shipped defaults, and this file is public. Change + them. +- `ADAPTER_IMAGE`, `DISCOVERY_IMAGE`, `MOCKIMD_IMAGE`, `MOCKAGMARKNET_IMAGE` — + the tags published for this environment. `TAG` pins discovery on its own: + `TAG=v0.3.1 make up` deploys a known build instead of whatever `latest` + points at today. Nothing is built here; everything is pulled. -Then: - -```sh -# 1. everything EXCEPT the adapters. Their configs do not exist yet, and -# step 2 is what writes them. -docker compose up -d registry discovery +The rest has working defaults and is commented where the reasoning is not +obvious. -# 2. generate the adapter keypairs, register the three adapter identities, -# render the three adapter configs -python3 bin/setup.py +`make up` runs five steps in the order they have to happen. `make up-core` +stops after step 3, which is enough to exercise the stack: -# 3. now the adapters -docker compose up -d - -# 4. the edge and the telemetry stack, both opt-in -docker compose --profile gateway --profile observability up -d +``` +1. registry and discovery (also registry-db, keycloak, discovery-db) +2. bin/setup.py keys, five registry participants, adapter configs +3. mocks, then the three adapters +4. nginx-proxy-manager the public edge -- 80 and 443, all interfaces +5. hyperdx ClickStack ``` -**Do not run a bare `docker compose up -d` for step 1.** An adapter config is -a bind-mounted *file*, and Docker creates a *directory* at any bind-mount -source that is missing. Starting an adapter early therefore wedges it on a -directory it cannot parse — `adapter.yaml: is a directory` — and leaves a -directory where step 2 needs to write a file. `bin/setup.py` refuses with an -explanation if it finds one; delete the empty directories and re-run. +Step 2 is the one to understand. It generates a keypair per adapter into +`keys/keys.json`, writes five participants and two capability bindings into +the registry, and renders the three adapter configs from the templates in +`config/adapters/`. **Nothing has to be created by hand afterwards** — and +nothing can be, from outside the VM, because the registry has no route. + +Step order is not cosmetic. An adapter config is a bind-mounted *file*, and +Docker creates a *directory* at any bind-mount source that is missing — so an +adapter started before step 2 wedges on `adapter.yaml: is a directory` and +leaves a directory where step 2 needs a file. This is the entire reason the +Makefile exists rather than a line in the README saying "run these in order". +`make up` gets it right; a bare `docker compose up -d` on a fresh checkout +does not. `bin/setup.py` refuses with an explanation if it finds one of those +directories — delete them and re-run. + +Re-running `make up` is safe. `setup.py` reuses the keys in `keys/keys.json` +and skips registry rows that already exist, so it converges rather than +failing on the second run. Check it: ```sh -docker compose ps +make ps curl -s -X POST http://127.0.0.1:8081/api/v1/Participant/search \ -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool ``` -Three participants, one per adapter. That is what `setup.py` seeded. +Five participants: three adapters and two upstreams. That is what `setup.py` +seeded, and that curl only works on the VM itself or through a tunnel. And through the gateway, once the proxy hosts exist: @@ -414,24 +443,60 @@ That 403 is the check worth repeating after any NPM change: it is the only evidence that `npm-custom/server_proxy.conf` is still mounted, and losing the mount silently opens an unauthenticated catalogue write. -## Register the provider +## What is in the registry, and why you did not create it + +`bin/setup.py` wrote all of it. Nothing in this section is a step to perform — +it is what to look at when something does not match. -**Quickest path: import `postman-collection/`.** It does everything in this -section and the end-to-end test after it — a token, the provider's two rows, -both registry searches, publish, discover and select — with every value -prefilled to match this deployment. Set one variable, `upstreamBaseUrl`, to a -URL the VM can reach for your API, and run the requests in order. +**Three `node` rows, one per adapter.** These are network identities: an id, a +role, and the public halves of a keypair. The private halves stay in +`keys/keys.json` on the VM and are never in the registry. A signature between +adapters is verified against these rows. + +Roles are `consumer`, `provider` and `network`, and they apply to `node` rows +only. A node needs at least one key, published as bare base64 with no encoding +label in front of it. + +**Two `upstream` rows, one per mock API.** An upstream is an ordinary HTTP API +this deployment calls. It signs nothing and nothing verifies it, so it needs no +role and no keys. It holds a `baseUrl` — here a compose service name, because +these are reached from inside the network and nowhere else. + +No credential for an upstream lives in the registry either. The adapter +presents credentials from its own config, which names *environment variables* +rather than values: the mandi binding uses `queryValueEnv`, and `MANDI_TOKEN` +reaches the container as an env var. + +**Two `ProviderSchema` rows, one per capability.** This is the row that says +which upstream answers which capability and how to call it — method, path, +timeout, retries, and the URL of the mapping file. Its `bindingKey` is +`participantId|capabilityCode`: + +``` +mausamgram-mock|openagrinet:WeatherObservation +agmarknet-mock|openagrinet:MandiPrice +``` -The rest of this section is the same thing as curl, if you would rather see it -step by step. Two rows, both by hand, and both need a token. +Those two strings are the hinge of the whole thing. The provider adapter builds +the same key out of each incoming payload — the provider id and the capability +`@type` it carries — and a step answers only when the key it was configured +with matches. `setup.py` renders those keys into `config/adapters/provider.yaml` +from the same `.env` values it seeds the registry from, which is what stops the +two from drifting. -Get the upstream API's URL first. If it is tunnelled from a laptop: +### Looking at it + +Only from the VM, or through a tunnel: ```sh -ngrok http 9100 +curl -s -X POST http://127.0.0.1:8081/api/v1/Participant/search \ + -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool + +curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema/search \ + -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool ``` -Take the `https://` URL. Then get a token: +Search takes no token. Writes do, and the token request has a trap in it: ```sh TOKEN=$(curl -s -X POST \ @@ -443,80 +508,52 @@ TOKEN=$(curl -s -X POST \ ``` Those two `X-Forwarded-*` headers are not optional, and `keycloak:8080` is the -**container-internal** address on purpose — not whatever `KEYCLOAK_PORT` is -published as. Keycloak builds the token's issuer from these headers, and the +**container-internal** address on purpose — not whatever `KEYCLOAK_PORT` +publishes it as. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and the registry rejects the token with a 401 and an empty body. -**Row one — the API itself.** Type `upstream`: an ordinary HTTP API this -deployment calls. It does not sign anything and nothing verifies it, so no -keys are needed — the signing in this flow is between adapters, on the three -`node` identities `bin/setup.py` seeded. `role` and `keys` are accepted on an -upstream if a deployment wants to record them; nothing reads them. - -```sh -curl -s -X POST http://127.0.0.1:8081/api/v1/Participant \ - -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ - -d '{ - "participantId": "my-weather-api", - "name": "My weather API", - "type": "upstream", - "status": "active", - "baseUrl": "https://YOUR-TUNNEL-SUBDOMAIN.ngrok-free.app" - }' -``` +### Pointing a capability at a real API -**Row two — which capability it answers, and how to call it.** +Two `.env` values and a re-run. To swap the weather mock for something real: ```sh -curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema \ - -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ - -d '{ - "bindingKey": "my-weather-api|openagrinet:WeatherObservation", - "participantId": "my-weather-api", - "capabilityCode": "openagrinet:WeatherObservation", - "status": "active", - "actions": [{ - "action": "select", - "method": "GET", - "path": "/get-daily", - "mappings": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml", - "timeoutMs": 15000, - "retryMax": 2, - "status": "active" - }] - }' +PROVIDER_PARTICIPANT_ID=imd-mausamgram # a new id, not the mock's +MAUSAMGRAM_BASE_URL=https://the-real-api.example.gov.in +MAUSAMGRAM_PATH=/the/real/path ``` -Things worth knowing about these two calls: +then `python3 bin/setup.py && docker compose up -d --force-recreate provider-adapter`. +That creates a new participant and a new binding, and re-renders the provider +config so its binding key matches. The old rows stay — see append-only below — +and become dead weight rather than a problem, since nothing sends their key. + +Things worth knowing before editing any of this: -- **No `{"Participant": {...}}` wrapper.** The registry takes the record - itself. A wrapper comes back as `extraneous key [Participant] is not - permitted`. -- **An `upstream` needs no `role` and no `keys`**, and no credential is held - for it here. It has never heard of Beckn, and nothing in the registry is - sent to it — the adapter presents credentials from its own config, naming - environment variables. `role` and `keys` are permitted if a deployment wants - to record them, but nothing reads them: a signature is verified against the - `node` identity that signed it. -- **The three roles are `consumer`, `provider` and `network`**, and they apply - to `node` rows only — the three `setup.py` creates. A node also needs at - least one key, published as bare base64 with no encoding label in front of - it. -- **`bindingKey` is `participantId|capabilityCode`.** It has to match what - `PROVIDER_PARTICIPANT_ID` and `PROVIDER_CAPABILITY` were set to in `.env` - when `setup.py` last ran — see the troubleshooting section for what a - mismatch answers. -- **`path` must start with one `/` and contain no empty segment.** The schema - refuses `//get-daily`, and so does the adapter. - **This registry is append-only.** There is no update, delete is soft, and a soft-deleted id keeps the unique index — so an id can never be reused. Got a - row wrong? Pick a new id. + row wrong? Pick a new id. This is why `PROVIDER_SUBSCRIBER_ID` and friends + are worth naming deliberately the first time. +- **Change one side of a binding key only and it fails**, in one of two ways + depending on which side. The troubleshooting section has both. +- **`path` must start with one `/` and contain no empty segment.** The schema + refuses `//get-daily`, and so does the adapter. +- **No `{"Participant": {...}}` wrapper** on a write. The registry takes the + record itself; a wrapper comes back as `extraneous key [Participant] is not + permitted`. +- **Registry schemas are read at startup.** Editing anything in + `config/registry/schemas/` needs `docker compose restart registry` before it + takes effect. ## Test it end to end -Replace `my-weather-api` if a different id was used, and point the coordinates -at wherever the API has data. +**Quickest path: import `postman-collection/`.** Six requests, 32 assertions, +nothing to fill in — publish, discover and select for both capabilities, with +every value already matching this deployment. A green run means the stack is +healthy rather than merely answering. + +The rest of this section is one of those requests as curl, if you would rather +see it than run it. ```sh curl -s -X POST http://127.0.0.1:9202/oan/select \ @@ -527,42 +564,58 @@ curl -s -X POST http://127.0.0.1:9202/oan/select \ "networkId": "oan-dev", "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", - "timestamp": "2026-09-02T06:12:01.330Z" + "timestamp": "2026-09-04T06:12:01.330Z" }, "message": { "contract": { "commitments": [ { "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, "resources": [ { - "id": "res:point-forecast", + "id": "res:mausamgram:point-forecast", "quantity": 1, "resourceAttributes": { "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", "subjectCategories": ["Weather"], + "informationMode": "OnDemand", + "supportedObservationTypes": ["Forecast"], + "supportedParameters": ["Rainfall", "Temperature"], + "geographicGranularities": ["Point"], "location": { "type": "Point", "coordinates": [73.7898, 19.9975] } } } ], "offer": { - "id": "offer:open-data", - "resourceIds": ["res:point-forecast"], - "provider": { "id": "my-weather-api", - "descriptor": { "code": "MY-API-01", "name": "My weather API" } } + "id": "offer:mausamgram:open-data", + "resourceIds": ["res:mausamgram:point-forecast"], + "provider": { "id": "mausamgram-mock", + "descriptor": { "code": "IMD-NWP-01", "name": "IMD Mausamgram NWP" } } } } ] } } }' | python3 -m json.tool ``` -You should get an `on_select` back, with one resource per forecast day. +An `on_select` comes back with one resource per forecast day — three by +default, which is `MOCKIMD_DAYS`. -No party is named in the payload, in either direction. Identity travels in -the `Authorization` header's `keyId`, which names the signer and the key the +The mandi equivalent is the same call to the same endpoint with a `MandiPrice` +resource and `agmarknet-mock` as the provider, and that is the point worth +taking from this section: **one endpoint, two capabilities, and no routing +config in between.** Each provider step builds a binding key out of the +payload it is handed, answers if the key is its own, and passes the payload +through untouched if it is not. Adding a third capability is a plugin and two +registry rows, not a new route. + +Two things about the payload: + +**No party is named, in either direction.** Identity travels in the +`Authorization` header's `keyId`, which names the signer and the key the registry published for it; a body that declares no caller simply skips the -declared-identity comparison. Nothing needs `bapId` or `bppId`, and the -`*Uri` fields they came with were container-internal addresses that meant -nothing outside this compose network anyway. +declared-identity comparison. Nothing needs `bapId` or `bppId`, and the `*Uri` +fields they came with were container-internal addresses that meant nothing +outside this compose network anyway. -The experience adapter is the only one that takes an unsigned request — the +**The experience adapter is the only one that takes an unsigned request.** The experience app is inside the trust boundary, so there is no network signature -to check. That is what makes this testable with a plain curl. +to check — which is what makes this testable with a plain curl. The same call +to the provider adapter on 9200 is rejected unsigned. ## Telemetry @@ -601,14 +654,22 @@ Three paths, and which adapter answers is the whole design: ``` discover you -> exp -> network -> discovery service -select you -> exp -> provider -> your upstream API -publish your catalogue system -> provider -> network -> discovery service +select you -> exp -> provider -> the upstream that owns that capability +publish a catalogue system -> provider -> network -> discovery service ``` `discover` and `publish` both end at the discovery service, and both go through the network adapter — that adapter is what fronts discovery, verifies the caller and re-signs. `select` never touches it: it goes straight to the -provider adapter, which answers from your upstream API. +provider adapter, which calls the upstream. + +Which upstream is not in any routing table. The provider adapter runs a chain +of capability steps — weather, then mandi — and each one builds a binding key +from the payload it is handed, serves the request if the key is its own, and +passes it along untouched if not. The step that claims it looks the upstream up +in the registry by that key. So one adapter fronts both capabilities, and a +third is a plugin plus two registry rows rather than a new route or a new +port. Each adapter's Beckn surface is one subtree, `/oan/`, and the payload's `action` says which action it is. That is the path the registry publishes as @@ -657,13 +718,15 @@ Two consequences worth knowing before you write a payload: a defect upstream, not something this deployment chose. Any value satisfies it. Without one, every `select` is refused with `SCH_REQUIRED_FIELD_MISSING: property "quantity" is missing`. -- **`publish` is not validated, because the spec does not define it.** The - validator refuses an action it cannot find with `unsupported action: publish`, - so the two modules that carry publishing — the provider adapter's root mount - and the network adapter — declare the validator but do not run it. To - validate publishing, give the validator an auxiliary spec that defines the - action: `auxiliaryTypes` and `auxiliaryLocations`, which are additive and - must not overlap the primary spec. +- **`publish` is not validated here, though it could be.** The two modules + that carry publishing — the provider adapter's root mount and the network + adapter — declare the validator but leave `validateSchema` out of their + `steps:`, and a plugin that is not in `steps:` never runs. That is a choice + in this config, not a limitation: the spec does define `/catalog/publish`, + the validator indexes it under the action `catalog/publish` that these + payloads send, and the collection's two publish bodies validate against it + with no errors. Turning it on is one line per module. It is off pending a + test rather than because it cannot work. An action the spec does not know, or a body missing a required field, comes back as a signed NACK with a `SCH_*` code and the JSON path that failed. @@ -675,7 +738,12 @@ docker-compose.yml the whole stack. Read it in tiers -- the banner comments are the structure: registry, discovery, adapters, observability (profile), edge (profile) .env.example copy to .env -bin/setup.py keys, the three adapter rows, the adapter configs +Makefile the front door: make up / up-core / down / help. +bin/ + bootstrap-ubuntu.sh docker and python on a fresh Ubuntu VM + stack.sh the startup order, and why it is that order. + Every make target is one line of delegation here. + setup.py keys, five registry rows, the adapter configs config/ gateway/ npm-custom/ mounted to /data/nginx/custom, which NPM includes @@ -703,71 +771,112 @@ config/ discovery/ instance.yaml.example optional override; see the compose file mappings/ - mausamgram/ the request and response transformation + mausamgram/ one file per binding-action: the request and the + agmarknet/ response transformation, in JSONata. These are the + files the adapters fetch over the raw CDN -- the + served copy and the reviewable copy are one file +mocks/ + mockimd/ the two mock upstreams. Sources only: they are + mockagmarknet/ pulled as published images like everything else. + See mocks/README.md for the build commands and + for what each deliberately gets wrong. postman-collection/ the whole flow as a Postman collection, with the - deployment's own values prefilled + deployment's own values prefilled and no registry + request in it +keys/keys.json generated, gitignored. The private halves of the + three adapter keypairs -- the one file here that + is worth backing up, and the reason setup.py can + be re-run without invalidating what it registered ``` -## About the mapping file +## About the mapping files -`config/mappings/` holds the mapping this deployment uses, and `MAPPING_URL` -points at **this repo's own copy** over GitHub's raw CDN. So the file a reader -reviews and the file the adapter fetches are one file, and cannot drift. +`config/mappings/` holds the two this deployment uses — one per binding-action +— and `MAPPING_URL` and `MANDI_MAPPING_URL` point at **this repo's own copies** +over GitHub's raw CDN. So the file a reader reviews and the file the adapter +fetches are one file, and cannot drift. + +Each file has two halves. The request half turns the incoming Beckn payload +into the query string or body the upstream expects; the response half turns +what comes back into the resources that go in the answer. The mandi one is the +better example of why this is not a field-renaming exercise: it converts ISO +dates to the `dd-MM-yyyy` Agmarknet wants, sends `marketcode` only when the +request carried one, turns price strings into numbers, and omits a price that +was not reported rather than sending a zero. It is a URL rather than a path because the registry publishes the full URL and the adapter fetches it verbatim — which means a mapping has to be reachable before it can be tested, and what this stack exercises is exactly what any consumer fetches. -Note the branch in that URL. Once this merges, point it at the default branch, -or pin a tag so a deployment is not following a moving file. +Note the branch in those URLs. Once this merges, point them at the default +branch, or pin a tag so a deployment is not following a moving file. -To change the mapping: edit the file here and push, or publish a fork anywhere -that serves raw text over https and put that URL in the `mappings` field of -the ProviderSchema row. +**What can be fixed here without touching code.** Quite a lot, and this is the +design intent: when a real upstream turns out to answer with different field +names, a different date format, or a nested envelope, that is a mapping edit +and a cache expiry. What is *not* fixable here is anything that depends on the +response never arriving — a non-2xx never reaches the mapping, because the +step fails first. -The adapter caches a mapping for `cacheTTL` (one minute, in the adapter config) -and GitHub's raw CDN caches for about five, so give an edit a few minutes to -show up. +To change one: edit the file here and push, or publish a fork anywhere that +serves raw text over https and put that URL in the `mappings` field of the +ProviderSchema row. The adapter caches a mapping for `cacheTTL` (one minute, +in the adapter config) and GitHub's raw CDN caches for about five, so give an +edit a few minutes to show up. ## When it does not work +Both of the common failures are a binding key disagreeing with itself, and +which 404 you get says which side is wrong. + **404 `NET_ENTITY_NOT_FOUND`, "this module serves no capability matching the -request".** The commonest one. The provider adapter did not recognise the -request as its own, so it passed it through and nothing behind it answered. +request".** No provider step recognised the request as its own, so each passed +it through and nothing behind them answered. -It decides that by building a binding key from the incoming payload — the +A step decides that by building a binding key from the incoming payload — the provider id at `message.contract.commitments[].offer.provider.id` and the capability at `...resources[].resourceAttributes.@type` — and comparing it -against the keys in its own config, which `setup.py` rendered from -`PROVIDER_PARTICIPANT_ID|PROVIDER_CAPABILITY`. - -Passing through is deliberate: it is what lets one adapter host several -capabilities. Compare all three — the payload, the `ProviderSchema` row, and -`.env` — and re-run `bin/setup.py` after changing `.env`. +against the key in its own config, which `setup.py` rendered from `.env`. -While the adapter carries one configured key, onboarding a second provider is -an edit to `.env`, a re-run of `bin/setup.py` and a restart of the provider -adapter. A registry entry on its own is not enough. +Passing through is deliberate: it is what lets this one adapter serve both +weather and mandi. Compare the payload against `.env`, and re-run +`bin/setup.py` plus `docker compose up -d --force-recreate provider-adapter` +after changing `.env`. -**502 with an empty body.** The adapter *is* configured for the key, but the -registry has no matching `ProviderSchema` row, so no call plan resolves. -Nothing in the response says so — the reason is in the log: +**404 naming a binding with no active record.** The other side. A step *is* +configured for the key, and it got as far as asking the registry which upstream +answers it — but there is no active `ProviderSchema` row with that +`bindingKey`, so no call plan resolves. ```sh -docker compose logs provider-adapter | grep "no call plan" +curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema/search \ + -H 'Content-Type: application/json' -d '{"filters":{}}' \ + | python3 -c 'import json,sys; [print(r["bindingKey"], r.get("status")) for r in json.load(sys.stdin)]' ``` -Check the row exists and that its `bindingKey` matches character for -character. The registry is append-only, so a mistyped row cannot be edited — -only superseded under a new id. +Compare character for character. The registry is append-only, so a mistyped +row cannot be edited — only superseded under a new id. (This used to be a 500 +with the reason only in the log; it is a 404 that names the binding now.) + +**A 502 from a select, with an upstream status in it.** Not a binding problem: +the upstream itself answered non-2xx. The step reports 4xx immediately and +retries 5xx up to `retryMax` from the `ProviderSchema` row. Credentials are a +likely cause — the mandi mock answers 401 without a token, which is what +`MANDI_TOKEN` is for. The log line carries the redacted URL. + +**Adding a third capability**, for reference, is a plugin in the adapter image, +one more entry under `providerSteps` *and* in `steps:` in the template, and two +registry rows. Declaring a step without adding its id to `steps:` is the quiet +failure mode: it never runs, and the request passes through to the 404 above. **The adapters restart in a loop on the first `up`.** Expected before -`bin/setup.py` has run — there is no `config/adapters/*.yaml` yet. +`bin/setup.py` has run — there is no `config/adapters/*.yaml` yet. `make up` +sequences this correctly; a bare `docker compose up -d` does not. -**`setup.py` says the registry did not come up.** Check `docker compose ps`. -The registry waits on Keycloak, which waits on Postgres, so a cold start takes -a minute or two. +**`setup.py` says the registry did not come up.** Check `make ps`. The registry +waits on Keycloak, which waits on Postgres, so a cold start takes a minute or +two — the healthcheck allows five. **`setup.py` says a participant is registered with a different key.** There is a `keys/keys.json` that no longer matches the registry. Restore the old one, or @@ -777,17 +886,15 @@ pick new `*_SUBSCRIBER_ID` values in `.env` — the old ids cannot be reused. minted for a different issuer than the registry validates against. Check the `X-Forwarded-Host` header is `keycloak:8080` and not the published port. -**A build fails on `go mod download`, or the adapter cannot fetch the mapping, -with "network is unreachable".** The host advertises IPv6 but cannot route it. -Add this to the adapter service in the compose file: +**A `docker compose pull` or a mapping fetch fails with "network is +unreachable".** The host advertises IPv6 but cannot route it. Add this to the +service in question: ```yaml sysctls: - net.ipv6.conf.all.disable_ipv6=1 ``` -and, if the build itself is what fails, `network: host` under its `build:`. - ### The gateway will not start ```sh diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index 1aabf3b..9d63533 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -1,230 +1,81 @@ { "info": { - "name": "OAN dev \u2014 registry to select", - "description": "The whole flow against a docker-deployment stack: register a provider, look at what the registry holds, publish a catalogue, discover it, and select from it.\n\nSET ONE VARIABLE AND RUN. Everything is prefilled in the collection's own variables, including providerId and capability as DEPLOYED -- so the binding key this creates already matches the adapter that will serve it. Only upstreamBaseUrl needs filling in: a URL the VM can reach for your API, which the deployment cannot know.\n\nWhy that matters. The provider adapter answers exactly one binding key, participantId|capabilityCode, rendered into its config from the stack's .env. A ProviderSchema row naming anything else is refused 404 'this module serves no capability matching the request'. providerId and capability here are prefilled to the values that deployment uses, so they agree by default -- change them only if the deployment changed.\n\nPREREQUISITE. bin/setup.py must have run on the stack. It seeds the three ADAPTER identities and renders their configs; this collection adds the provider, because the provider's base URL is not the stack's to know.\n\nRun in order the first time -- request 1 issues the token requests 2 and 3 need. Re-running is safe: the registry is append-only, so a repeat create reports the existing row rather than changing anything.\n\nPorts bind to loopback on the VM, so from a workstation tunnel first:\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 you@the-vm", + "name": "OAN dev \u2014 publish, discover, select for two capabilities", + "description": "The whole flow against a docker-deployment stack: a publish, a discover and a select for each of the two capabilities.\n\nTHERE ARE NO REGISTRY REQUESTS HERE, and that is deliberate. The registry is not reachable from outside the stack -- no port beyond loopback, no proxy host in front of it -- so nothing in this collection could create or read a registry row. `bin/setup.py` seeds all of it: five participants and two capability bindings, from the same .env the adapter configs are rendered from, which is what keeps the two from disagreeing.\n\nSo the prerequisite is `make setup` followed by `make up`. After that everything here works with no value to fill in -- providerId and mandiProviderId are already the ids that deployment uses.\n\nRequests 5 and 6 are the point. They hit the same endpoint on the same adapter and different domain packages answer them, because each provider step recognises its own binding key from the payload and passes through anything else. Nothing routes by URL or by domain.\n\nRun in order the first time: 1 and 2 publish what 3 and 4 look for.\n\nThe ports are loopback, so from a workstation tunnel first:\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 you@the-vm", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "variable": [ - { - "key": "keycloakUrl", - "value": "http://127.0.0.1:8080" - }, - { - "key": "registryUrl", - "value": "http://127.0.0.1:8081/api/v1" - }, { "key": "expAdapterUrl", - "value": "http://127.0.0.1:9202" + "value": "http://127.0.0.1:9202", + "description": "The experience adapter -- the only one that takes an unsigned request." }, { "key": "providerAdapterUrl", - "value": "http://127.0.0.1:9200" - }, - { - "key": "keycloakRealm", - "value": "sunbird-rc" - }, - { - "key": "keycloakClientId", - "value": "registry-frontend" - }, - { - "key": "registryUser", - "value": "no-user" - }, - { - "key": "registryPassword", - "value": "no-user-password" + "value": "http://127.0.0.1:9200", + "description": "Where publish enters: the provider adapter signs it and forwards to the network layer." }, { "key": "providerId", - "value": "my-weather-api", - "description": "AS DEPLOYED: PROVIDER_PARTICIPANT_ID in the stack's .env. With this and capability below it forms the binding key the provider adapter answers to. Change only if that deployment was changed." - }, - { - "key": "capability", - "value": "openagrinet:WeatherObservation", - "description": "AS DEPLOYED: PROVIDER_CAPABILITY in the stack's .env." - }, - { - "key": "upstreamBaseUrl", - "value": "https://YOUR-TUNNEL-SUBDOMAIN.ngrok-free.app", - "description": "THE ONE VALUE TO SET. Your upstream API, reachable from the VM. The deployment cannot know it." - }, - { - "key": "upstreamPath", - "value": "/get-daily", - "description": "The path on your API for the select action. One leading slash, no empty segment." + "value": "mausamgram-mock", + "description": "AS DEPLOYED: PROVIDER_PARTICIPANT_ID in .env. Half of the weather binding key." }, { - "key": "mappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml", - "description": "Full URL of the request/response mapping, fetched verbatim by the adapter. This is the deployment's own copy in the helmcharts repo, so what the adapter fetches is the reviewed file. Note the branch in the path -- repoint it once that merges." + "key": "mandiProviderId", + "value": "agmarknet-mock", + "description": "AS DEPLOYED: MANDI_PARTICIPANT_ID in .env. Half of the mandi binding key." }, { "key": "networkId", - "value": "oan-dev" + "value": "oan-dev", + "description": "APP_NETWORK_ID in .env. Discovery scopes by this: a catalogue published under a different one is invisible to the query." }, { "key": "domain", - "value": "oan-dev" - }, - { - "key": "catalogId", - "value": "cat-weather-point-forecast" - }, - { - "key": "token", - "value": "", - "description": "Filled in by request 1." + "value": "oan-dev", + "description": "Also required for a discover to match." } ], "item": [ { - "name": "1. Registry \u2014 get a write token", - "request": { - "method": "POST", - "header": [ - { - "key": "X-Forwarded-Host", - "value": "keycloak:8080" - }, - { - "key": "X-Forwarded-Proto", - "value": "http" - } - ], - "url": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", - "description": "Keycloak issues the bearer token the two registry writes below need.\n\nThe two X-Forwarded-* headers are not optional. Keycloak runs behind PROXY_ADDRESS_FORWARDING and builds the token's issuer from them, and the registry validates that issuer against keycloak:8080 -- the CONTAINER-INTERNAL address, not whatever KEYCLOAK_PORT is published as. Get it wrong and every write is refused 401 with an empty body.\n\nThe token is saved to the {{token}} collection variable.", - "body": { - "mode": "urlencoded", - "urlencoded": [ - { - "key": "client_id", - "value": "{{keycloakClientId}}" - }, - { - "key": "grant_type", - "value": "password" - }, - { - "key": "username", - "value": "{{registryUser}}" - }, - { - "key": "password", - "value": "{{registryPassword}}" - } - ] - } - }, - "response": [], - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const j = pm.response.json();", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", - "pm.collectionVariables.set(\"token\", j.access_token);" - ] - } - } - ] - }, - { - "name": "2. Registry \u2014 create the provider", + "name": "1. Publish \u2014 weather catalogue", "request": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant", - "description": "The upstream API itself, and it deliberately carries NO KEYS.\n\nWHERE ARE THE SIGNING KEYS, THEN. The registry holds two kinds of participant, and only one of them signs:\n\n type \"node\" a Beckn participant. REQUIRES role and keys. These are the three adapters -- exp, network and provider -- and bin/setup.py created them, generating a keypair for each, publishing the public half here and rendering the private half into that adapter's own config. They are what signAck signs with and what validateSign verifies against.\n\n type \"upstream\" an ordinary HTTP API our provider adapter calls. It has never heard of Beckn: it does not sign its responses and nothing verifies them. role and keys are permitted on it but not expected, and nothing in the adapter reads them: a signature is verified against the node identity that signed it, never against an upstream.\n\nSo the signing in this flow is entirely between adapters, on the identities setup.py seeded. Run request 4 to see all four rows side by side: three nodes with keys, this one without.\n\nCredentials for calling this API, if it needs any, are not held here either -- the provider adapter's own config names the environment variables they come from, so nothing secret is in the registry.\n\nSet upstreamBaseUrl to a URL the VM can reach; an ngrok https URL if the API runs on a laptop.\n\nNo {\"Participant\": {...}} wrapper: the registry takes the record itself, and a wrapper is refused as 'extraneous key [Participant] is not permitted'.\n\nThe registry is append-only. There is no update, delete is soft and keeps the unique index, so a participantId can never be reused -- get this wrong and pick a new id.", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"Weather API\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{upstreamBaseUrl}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - } - }, - "response": [], - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const p = pm.response.json().params;", - "pm.test(\"the registry answered (created, or already present)\", () => pm.expect([\"SUCCESSFUL\",\"UNSUCCESSFUL\"]).to.include(p.status));", - "if (p.status !== \"SUCCESSFUL\") {\n // Expected on a re-run: the registry is append-only, so a second create of\n // the same id is a duplicate-key failure, not a problem. Anything else is.\n console.log(\"not created -- already present, or refused:\", p.errmsg);\n}" - ] - } - } - ] - }, - { - "name": "3. Registry \u2014 create the capability binding", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/ProviderSchema", - "description": "Which capability this provider answers, and how to call it.\n\nbindingKey is {{providerId}}|{{capability}}, and both are prefilled to what the adapter was deployed with -- so this row matches by default. It is one key: while the adapter carries a single configured key this is a one-provider deployment, and onboarding another means editing .env, re-running bin/setup.py and restarting the provider adapter. A registry entry on its own is not enough.\n\nHow a mismatch shows up:\n payload names a provider the adapter is not configured for\n 404 NET_ENTITY_NOT_FOUND, 'this module serves no capability matching the request'\n adapter configured for the key but this row missing\n 502 with an empty body; the reason is only in\n docker compose logs provider-adapter | grep 'no call plan'\n\npath must start with one / and carry no empty segment; //get-daily is refused by the schema and by the adapter. mappings is the FULL url of the mapping file, fetched verbatim.\n\nNo {\"ProviderSchema\": {...}} wrapper -- the registry takes the record itself.", + "url": "{{providerAdapterUrl}}/publish", "body": { "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{providerId}}|{{capability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{capability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{upstreamPath}}\",\n \"mappings\": \"{{mappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } - } + }, + "description": "Enters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service. Posting straight at the discovery service would skip both adapters, and so skip the part worth testing.\n\ncontext.action is catalog/publish, the name the Beckn v2 spec gives this action at /catalog/publish. A bare \"publish\" is not a spec action. The callback is catalog/on_publish.\n\nThe caller signs nothing and the body names no party: identity travels in the Authorization header's keyId, from the adapter's own keyManager config.\n\nThe catalogue carries no offers -- nothing requires them, and select does not read them: it carries its own offer in the request." }, - "response": [], "event": [ { "listen": "test", "script": { "type": "text/javascript", "exec": [ - "const p = pm.response.json().params;", - "pm.test(\"the registry answered (created, or already present)\", () => pm.expect([\"SUCCESSFUL\",\"UNSUCCESSFUL\"]).to.include(p.status));", - "if (p.status !== \"SUCCESSFUL\") {", - " // Expected on a re-run: the registry is append-only, so a second create of", - " // the same id is a duplicate-key failure, not a problem. Anything else is.", - " console.log(\"not created -- already present, or refused:\", p.errmsg);", - "}", - "console.log(\"binding key created:\", pm.variables.get(\"providerId\") + \"|\" + pm.variables.get(\"capability\"),", - " \"-- this must be what the provider adapter was deployed with\");" + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", + "pm.test(\"ACCEPTED\", () => pm.expect(b.message.results[0].status).to.eql(\"ACCEPTED\"));" ] } } ] }, { - "name": "4. Registry \u2014 search participants", + "name": "2. Publish \u2014 mandi catalogue", "request": { "method": "POST", "header": [ @@ -233,11 +84,11 @@ "value": "application/json" } ], - "url": "{{registryUrl}}/Participant/search", - "description": "What the registry now holds, and the answer to 'where are the keys'.\n\nExpect four rows: the three ADAPTER identities that bin/setup.py seeded, each type \"node\" with a role and a published signing key, and the provider created above, type \"upstream\" with neither. That split is the whole trust model -- adapters sign and are verified against these published keys; the upstream API is just an HTTP endpoint behind one of them.\n\nNote the key shape on a node: bare base64, no encoding label, identified by the osid the registry assigned on write, and no keyId or use field.", + "url": "{{providerAdapterUrl}}/publish", + "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8.", "body": { "mode": "raw", - "raw": "{\n \"filters\": {}\n}", + "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -252,34 +103,19 @@ "script": { "type": "text/javascript", "exec": [ - "const rows = pm.response.json().data;", "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const mine = rows.filter(r => r.participantId === pm.variables.get(\"providerId\"));", - "pm.test(\"the provider is there\", () => pm.expect(mine.length).to.eql(1));", - "pm.test(\"it is an upstream, with no role and no keys\", () => {", - " pm.expect(mine[0].type).to.eql(\"upstream\");", - " pm.expect(mine[0].role).to.be.undefined;", - " pm.expect(mine[0].keys).to.be.undefined;", - "});", - "", - "// The other side of the trust model: the adapters DO publish keys, and those", - "// are what every signature in this flow is verified against. If these are", - "// missing, bin/setup.py has not run and nothing will authenticate.", - "const nodes = rows.filter(r => r.type === \"node\");", - "pm.test(\"the adapter identities are seeded, each with a signing key\", () => {", - " pm.expect(nodes.length).to.be.at.least(3);", - " nodes.forEach(n => {", - " pm.expect(n.role, n.participantId + \" has no role\").to.be.a(\"string\");", - " pm.expect((n.keys || []).length, n.participantId + \" publishes no key\").to.be.above(0);", - " });", - "});" + "const b = pm.response.json();", + "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", + "const r = b.message.results[0];", + "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", + "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(\"cat-agmarknet-mandi-prices\"));" ] } } ] }, { - "name": "5. Registry \u2014 search provider bindings", + "name": "3. Discover \u2014 weather", "request": { "method": "POST", "header": [ @@ -288,35 +124,40 @@ "value": "application/json" } ], - "url": "{{registryUrl}}/ProviderSchema/search", - "description": "The capability bindings. One row per provider per capability, each carrying the call plan the provider adapter resolves against at request time -- which is why repointing a provider at a new URL is a registry write and not a config change.", + "url": { + "raw": "{{expAdapterUrl}}/oan/discover", + "host": [ + "{{expAdapterUrl}}/oan/discover" + ] + }, "body": { "mode": "raw", - "raw": "{\n \"filters\": {}\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", "options": { "raw": { "language": "json" } } - } + }, + "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." }, - "response": [], "event": [ { "listen": "test", "script": { "type": "text/javascript", "exec": [ - "const rows = pm.response.json().data;", - "const want = pm.variables.get(\"providerId\") + \"|\" + pm.variables.get(\"capability\");", - "pm.test(\"the binding is there\", () => pm.expect(rows.map(r => r.bindingKey)).to.include(want));" + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", + "pm.test(\"at least one catalogue\", () => pm.expect((b.message.catalogs || []).length).to.be.above(0));" ] } } ] }, { - "name": "6. Publish \u2014 one catalogue, one resource", + "name": "4. Discover \u2014 mandi", "request": { "method": "POST", "header": [ @@ -325,11 +166,11 @@ "value": "application/json" } ], - "url": "{{providerAdapterUrl}}/publish", - "description": "context.action is catalog/publish, which is the name the Beckn v2 spec gives this action at /catalog/publish -- a bare \"publish\" is not a spec action and would be refused by schema validation. The callback comes back as catalog/on_publish.\n\nEnters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service.\n\nThe caller signs nothing, and the body names no party: identity travels in the Authorization header's keyId, taken from the adapter's own keyManager config.\n\n/publish sits at the root, outside /oan/ -- it is not part of this adapter's Beckn surface, so it must not shadow it.\n\nThe catalogue carries no offers. Nothing requires them: a catalogue of resources alone is legal, and the discovery service stores the offers member only when one is sent. select does not read them either -- it carries its own offer in the request, and the answer echoes that one back with its resource references rewritten.", + "url": "{{expAdapterUrl}}/oan/discover", + "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing.", "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-02T06:00:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"{{catalogId}}\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"WX-01\",\n \"name\": \"Point weather forecast catalogue\",\n \"shortDesc\": \"Daily point weather forecast\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, several days ahead.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"WX-01\",\n \"name\": \"Weather API\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Weather forecast for a single point\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"Alert\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", "options": { "raw": { "language": "json" @@ -345,16 +186,29 @@ "type": "text/javascript", "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const r = pm.response.json().message.results[0];", - "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", - "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(pm.variables.get(\"catalogId\")));" + "const b = pm.response.json();", + "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", + "const cats = b.message.catalogs || [];", + "pm.test(\"at least one catalogue\", () => pm.expect(cats.length).to.be.above(0));", + "// The mandi catalogue specifically, so this cannot pass on the weather one.", + "pm.test(\"the mandi catalogue is discoverable\", () => {", + " pm.expect(cats.map(c => c.id)).to.include(\"cat-agmarknet-mandi-prices\");", + "});", + "// A catalogue entry advertises a capability rather than carrying data.", + "pm.test(\"it advertises OnDemand and carries no prices\", () => {", + " const mandi = cats.find(c => c.id === \"cat-agmarknet-mandi-prices\");", + " const ra = mandi.resources[0].resourceAttributes;", + " pm.expect(ra.informationMode).to.eql(\"OnDemand\");", + " pm.expect(ra).to.not.have.property(\"prices\");", + " pm.expect(ra.supportedCommodities.length).to.be.above(0);", + "});" ] } } ] }, { - "name": "7. Discover \u2014 find it", + "name": "5. Select \u2014 weather, per-day forecast", "request": { "method": "POST", "header": [ @@ -363,36 +217,63 @@ "value": "application/json" } ], - "url": "{{expAdapterUrl}}/oan/discover", - "description": "Experience adapter -> network layer -> discovery service. The experience adapter is the only one that takes an unsigned request, because the app in front of it is inside the trust boundary -- which is what makes this callable with no signature.", + "url": { + "raw": "{{expAdapterUrl}}/oan/select", + "host": [ + "{{expAdapterUrl}}/oan/select" + ] + }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"domain\": \"{{domain}}\",\n \"transactionId\": \"3a7d9c11-2b4e-4f60-8a1c-5d6e7f809a1b\",\n \"messageId\": \"4b8e0d22-3c5f-4071-9b2d-6e7f8091a2b3\",\n \"timestamp\": \"2026-09-02T06:20:00.000Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } - } + }, + "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." }, - "response": [], "event": [ { "listen": "test", "script": { "type": "text/javascript", "exec": [ + "if (pm.response.code === 404) {", + " console.log(\"404: the provider adapter serves no capability matching this payload.\",", + " \"offer.provider.id and resourceAttributes.@type must equal the deployed\",", + " \"binding key -- check request 2 against PROVIDER_PARTICIPANT_ID in .env\");", + "}", + "", "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const m = pm.response.json().message;", - "pm.test(\"on_discover\", () => pm.expect(pm.response.json().context.action).to.eql(\"on_discover\"));", - "pm.test(\"at least one catalogue\", () => pm.expect((m.catalogs || []).length).to.be.above(0));" + "const b = pm.response.json();", + "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", + "const c = b.message.contract.commitments[0];", + "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", + "", + "// Spec conformance of the answer, which is the mapping's job and so the", + "// thing that silently regresses when the published mapping changes.", + "pm.test(\"status is in the spec enum\", () => {", + " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", + "});", + "pm.test(\"every resource carries a quantity\", () => {", + " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", + "});", + "pm.test(\"the offer references only resources returned\", () => {", + " const ids = c.resources.map(r => r.id);", + " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", + "});", + "pm.test(\"no party named in the answer\", () => {", + " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", + "});" ] } } ] }, { - "name": "8. Select \u2014 quote it", + "name": "6. Select \u2014 mandi, prices per market day", "request": { "method": "POST", "header": [ @@ -402,10 +283,10 @@ } ], "url": "{{expAdapterUrl}}/oan/select", - "description": "Experience adapter -> provider adapter -> the upstream API. Never touches the network layer.\n\nThe resource is a full openagrinet:WeatherObservation v0.1, in OnDemand mode -- which is what a select IS: asking a provider to obtain the information rather than carrying it. The pack requires informationMode and subjectCategories in either mode, and in OnDemand it requires supportedObservationTypes, supportedParameters and geographicGranularities while FORBIDDING parameters. So a request states what it wants observed, never the readings themselves.\n\nquantity is required on each resource by the pinned Beckn v2 spec, which defines no quantity property and has no Quantity schema -- a defect upstream. Any value satisfies it; without one, schema validation refuses the request.\n\nThe answer comes back in Direct mode with ONE RESOURCE PER FORECAST DAY, ids derived from each date, and the offer's resourceIds rewritten to match. Its context carries only correlation ids -- no party is named in either direction.", + "description": "The same endpoint as request 5, the same adapter, a different capability. Nothing routes this: the payload's provider id and resourceAttributes @type form a binding key, the mandi step recognises it and the weather step passes it through.\n\nThe answer is a Direct openagrinet:MandiPrice per price record. Its prices arrive from the upstream as STRINGS with Title Case keys containing spaces, so the mapping converts them; and a record that reported no minimum or maximum must come back with those absent rather than zeroed, which is what the last assertion checks.\n\nThe upstream's credential is a query parameter, which the adapter adds from an environment variable and redacts from the URL it logs.", "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-09-02T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:point-forecast\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:open-data\",\n \"resourceIds\": [\n \"res:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"WX-01\",\n \"name\": \"Weather API\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -420,31 +301,52 @@ "script": { "type": "text/javascript", "exec": [ - "// A 404 here is the binding-key mismatch, not a wiring problem: the response", - "// names the module, not the cause.", "if (pm.response.code === 404) {", - " console.log(\"404: the provider adapter serves no capability matching this payload.\",", - " \"offer.provider.id and resourceAttributes.@type must equal the deployed key:\",", - " pm.variables.get(\"providerId\") + \"|\" + pm.variables.get(\"capability\"));", - "}", - "// A 502 with no body is the other half: the adapter is configured for the key", - "// but no ProviderSchema row resolves. Only the adapter log says so.", - "if (pm.response.code === 502) {", - " console.log(\"502: no call plan resolved. Run request 5 to confirm the row exists, then\",", - " \"docker compose logs provider-adapter | grep 'no call plan'\");", + " console.log(\"404: no step matched. offer.provider.id and resourceAttributes.@type must\",", + " \"equal a configured binding key -- check MANDI_PARTICIPANT_ID in .env and\",", + " \"that mandi is in the provider adapter's steps: list, not just providerSteps\");", "}", "", "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", "const c = b.message.contract.commitments[0];", - "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", - "const ids = c.resources.map(r => r.id);", - "pm.test(\"the offer references only resources returned\", () => {", - " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", + "pm.test(\"a resource per price record\", () => pm.expect(c.resources.length).to.be.above(0));", + "", + "const first = c.resources[0].resourceAttributes;", + "pm.test(\"MandiPrice in Direct mode\", () => {", + " pm.expect(first[\"@type\"]).to.eql(\"openagrinet:MandiPrice\");", + " pm.expect(first.informationMode).to.eql(\"Direct\");", "});", - "pm.test(\"no party named in the answer\", () => {", - " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", + "// Direct requires all six of these in the pack.", + "pm.test(\"the pack's Direct fields are all present\", () => {", + " [\"source\",\"commodity\",\"market\",\"arrivalDate\",\"prices\",\"generatedAt\"].forEach(", + " f => pm.expect(first, f).to.have.property(f));", + "});", + "// The upstream sends prices as strings; the pack requires numbers.", + "pm.test(\"prices are numbers, not strings\", () => {", + " pm.expect(first.prices.modal).to.be.a(\"number\");", + " pm.expect(first.prices.currency).to.eql(\"INR\");", + "});", + "// dd-MM-yyyy upstream, ISO in the answer.", + "pm.test(\"arrivalDate is ISO\", () => {", + " pm.expect(first.arrivalDate).to.match(/^\\d{4}-\\d{2}-\\d{2}$/);", + "});", + "pm.test(\"status is in the spec enum\", () => {", + " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", + "});", + "pm.test(\"every resource carries a quantity\", () => {", + " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", + "});", + "// A record the market reported partially must come back partial, not zeroed:", + "// \"no minimum reported\" and \"a minimum of zero\" are different facts.", + "pm.test(\"an unreported price is absent, not zero\", () => {", + " const last = c.resources[c.resources.length - 1].resourceAttributes.prices;", + " if (c.resources.length > 1) {", + " pm.expect(last).to.not.have.property(\"minimum\");", + " pm.expect(last).to.not.have.property(\"maximum\");", + " }", + " pm.expect(last.modal).to.be.a(\"number\");", "});" ] } diff --git a/docker-deployment/postman-collection/README.md b/docker-deployment/postman-collection/README.md index 673134f..27848f9 100644 --- a/docker-deployment/postman-collection/README.md +++ b/docker-deployment/postman-collection/README.md @@ -1,32 +1,65 @@ # Postman collection -`OAN-dev-flow.postman_collection.json` runs the whole flow against a stack -brought up from the compose file beside it: a write token, the provider's two -registry rows, both registry searches, publish, discover and select. - -Import it and set **one** variable — `upstreamBaseUrl`, a URL the VM can reach -for your upstream API. Everything else is prefilled in the collection itself, -including `providerId` and `capability`, which are set to the values this -deployment's `.env.example` uses. So the binding key the collection creates -already matches the adapter that will serve it. - -Two things to know: - -- **`bin/setup.py` must have run first.** It seeds the three adapter - identities and renders their configs. The collection adds only the provider, - because the provider's base URL is not the stack's to know. -- **Run the requests in order the first time.** Request 1 issues the token that - requests 2 and 3 need. Re-running is safe: the registry is append-only, so a - repeat create reports the existing row rather than changing anything. - -The requests carry assertions, so a run tells you whether the stack is -actually healthy rather than just returning 200s. Among them: that the three -adapter identities exist with signing keys, that publish is `ACCEPTED`, that -`select` answers with a resource per forecast day, that its status is in the -spec's enum, and that every resource carries a `quantity`. - -If ports are bound to loopback on the VM, tunnel first and the defaults work -unchanged: - - ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ - -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 you@the-vm +`OAN-dev-flow.postman_collection.json` — publish, discover and select, for +each of the two capabilities, against a stack brought up from the compose file +beside it. Six requests, 32 assertions. + +Import it and run it. **There is nothing to fill in.** Every variable is +prefilled with what this deployment actually uses, including the two provider +ids, so the binding keys in the payloads already match the adapter that will +serve them. + +## There are no registry requests here + +Deliberately. The registry has no route through the gateway and publishes on +loopback only, so a Postman request could not create or read a row from +outside the VM. `bin/setup.py` seeds all of it — five participants and both +capability bindings — which is what makes this collection short. + +So the prerequisite is the stack being up the normal way: + + make up + +## Run them in order the first time + +Requests 1 and 2 publish the catalogues that 3 and 4 search for. After that +any request works on its own. Re-running is safe: publish is idempotent from +the caller's point of view, and nothing here writes to the registry. + +## What the assertions actually check + +Enough that a green run means the stack is healthy, not just answering: + +- publish comes back `ACCEPTED` +- discover returns at least one catalogue, and for mandi that the specific + catalogue just published is the one found +- a discovered catalogue advertises `OnDemand` and carries no prices — the + pack forbids them in that mode +- select answers with one resource per forecast day, and one per price record +- the mandi answer is in `Direct` mode with every field the pack requires of + it, its prices are numbers rather than the strings the upstream sends, and + `arrivalDate` is ISO rather than the `dd-MM-yyyy` that arrived +- a record with no minimum or maximum omits those fields instead of sending + nulls — the last mock record is built that way on purpose +- status codes come from the spec's `DRAFT|ACTIVE|CLOSED` enum +- every resource carries a `quantity` +- the weather answer names no party in either direction, and its offer + references only resources actually returned + +## Requests 5 and 6 are the interesting pair + +They hit **the same endpoint on the same adapter**, and different domain +packages answer them. Each provider step builds a binding key from the payload +it is given — provider id plus capability `@type` — serves it if the key is +its own, and passes it through untouched if not. Nothing routes by URL, by +path or by domain, which is what lets one adapter host both capabilities and +what makes adding a third a config change. + +## Tunnelling + +The ports are on the VM's loopback. From a workstation: + + ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 -N you@the-vm + +Those two are all the collection needs: 9202 is the experience adapter, 9200 +the provider adapter, which is where a publish enters. From e508c18b7c609c3bbc69995697880b4e02d40d84 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:02:12 +0530 Subject: [PATCH 31/81] docs: correct setup.py's own account of what it does [OpenAgriNet/network-adapter#4] The module docstring still said it does not register the provider and that those two rows are created by hand with the curl in README. It seeds all five participants and both bindings now, and the README no longer has a curl to follow. Rewrites the docstring around what it actually writes, and fixes three smaller things in the same file: the recovery hint pointed at a two-command sequence the Makefile replaced, the closing message told the reader to run `make up` next when `make up` is what invoked it, and one sentence called a binding-key mismatch quiet immediately before quoting the 404 it answers. --- docker-deployment/bin/setup.py | 68 +++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 5f53e19..6249a62 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -1,19 +1,39 @@ #!/usr/bin/env python3 -"""Prepare the stack: generate the adapter keypairs, register the three adapter -identities, render the adapter configs. +"""Prepare the stack: generate the adapter keypairs, seed the registry, render +the adapter configs. python3 bin/setup.py -Safe to re-run. Keys are generated once and reused from keys/keys.json, so the -identities already in the registry stay valid; participants that exist are left -alone rather than recreated, because this registry's delete is soft and holds -the unique index -- a deleted participantId cannot be reused. +`make up` runs this as step 2, which is where it belongs -- the adapter configs +it renders are bind-mounted files, and an adapter started before they exist +leaves a directory in their place. + +WHAT IT WRITES. Five participants and two capability bindings: + + 3 x node one per adapter -- exp, network, provider -- each with the + public halves of a keypair. The private halves stay in + keys/keys.json and never reach the registry. + 2 x upstream the two APIs this deployment calls, addressed by compose + service name. An upstream signs nothing, so it needs no role + and no keys. + 2 x binding a ProviderSchema row per capability: which upstream answers + it, the method and path, timeouts, and the mapping URL. + +This is all of it. Nothing has to be created by hand afterwards, and nothing +can be from outside the VM -- the registry has no route through the gateway and +publishes on loopback only, which is why seeding lives here rather than in a +Postman request. -WHAT THIS DOES NOT DO: it does not register the provider. The three rows here -are the adapters' own identities, which they need before they can sign anything -or verify each other. The upstream API is a Participant of type "upstream" plus -a ProviderSchema row, and its base URL belongs to whoever runs it -- so those -two are created by hand. README.md has the curl. +The binding keys are the load-bearing part. A provider step answers only when +the key it was configured with matches the one built from the incoming payload, +and both sides come from the same .env values in this one run -- which is what +keeps them from disagreeing. To point a capability somewhere else: edit .env, +re-run this, recreate the provider adapter. + +Safe to re-run. Keys are generated once and reused from keys/keys.json, so the +identities already in the registry stay valid; participants and bindings that +exist are left alone rather than recreated, because this registry's delete is +soft and holds the unique index -- a deleted participantId cannot be reused. Needs python3 and the cryptography package: @@ -372,7 +392,7 @@ def render(identities): f" script ran. Bring them down, remove the empty directories and retry:\n" f" docker compose down\n" f" rmdir config/adapters/*.yaml\n" - f" docker compose up -d registry discovery && python3 bin/setup.py") + f" make up") out.write_text(template) out.chmod(0o600) # holds a private key print(f" config/adapters/{role}.yaml") @@ -394,15 +414,21 @@ def render(identities): Those are the binding keys the provider adapter answers to. They were rendered into its config from the same .env this seeded the registry from, which is what -keeps the two from disagreeing -- and a disagreement is quiet: a payload naming -anything else is answered 404 "this module serves no capability matching the -request". +keeps the two from disagreeing. A payload naming anything else is answered 404 +"this module serves no capability matching the request" -- explicit, but it +names the request rather than the mismatch, so compare it against these two. Both providers are mocks reached by compose service name. Pointing a capability -at a real upstream is a registry write, not a change here: a new Participant -with its base URL, a ProviderSchema row naming it, and the adapter's -{env('PROVIDER_CAPABILITY')} or {env('MANDI_CAPABILITY')} entry in .env -updated to match. The registry is not reachable from outside this stack, so -that write happens from here. +at a real upstream is an .env edit and a re-run of this: a new participant id +and base URL under PROVIDER_* or MANDI_*, which seeds a new Participant and +ProviderSchema row and re-renders the provider config so its binding key +matches. The registry is not reachable from outside this stack, so that write +happens from here. + +Next: `make up` continues to step 3 and starts the adapters. If you ran this +on its own, the adapters need recreating to pick up the rendered configs: + + docker compose up -d --force-recreate provider-adapter network-adapter exp-adapter -Next: make up, then import postman-collection/ and run it.""") +Then import postman-collection/ and run it -- six requests, nothing to fill +in.""") From 2e19a93ee1acb483c51c0615b0d34d2b4a6fa693 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:28:56 +0530 Subject: [PATCH 32/81] feat: send telemetry from all three adapters to the collector [OpenAgriNet/network-adapter#4] The stack has shipped a collector since the observability tier went in, and nothing was sending to it. Each adapter now configures the otelsetup plugin and exports metrics, traces and logs over OTLP/gRPC to hyperdx:4317. It goes under the top-level plugins: key rather than inside a module. That is the application-plugin section -- things with their own background lifecycle, an exporter and a periodic flush, rather than something a step loads per request. Absent, it is skipped; there is no requirement to configure it. OTEL_ENABLED is one switch over all three signals, and it matters more than a tidy toggle. With every signal false the plugin builds no exporter and never dials, so a stack brought up with `make up-core` -- which does not start the observability profile -- stays quiet. Left true against a collector that is not running, the exporter retries on a loop and says so in the log every few seconds. Verified both paths against the published image: enabled logs "OpenTelemetry metrics initialized", disabled logs "metrics, tracing and logs are all disabled" and builds nothing. serviceVersion is deliberately not set. Left empty the plugin substitutes the adapter's own build version -- v1.8.2-106-gb8193d6 on the image tested -- which is truer than a string in a template and cannot go stale. The endpoint and environment are rendered from .env like everything else, because the adapter does not expand environment variables in its config: what setup.py writes is what it reads. --- docker-deployment/.env.example | 17 +++++++++ docker-deployment/bin/setup.py | 9 ++++- .../config/adapters/exp.yaml.tmpl | 36 +++++++++++++++++++ .../config/adapters/network.yaml.tmpl | 36 +++++++++++++++++++ .../config/adapters/provider.yaml.tmpl | 36 +++++++++++++++++++ 5 files changed, 133 insertions(+), 1 deletion(-) diff --git a/docker-deployment/.env.example b/docker-deployment/.env.example index de94fb1..d4390a5 100644 --- a/docker-deployment/.env.example +++ b/docker-deployment/.env.example @@ -72,6 +72,23 @@ REGISTRY_DEFAULT_USER_PASSWORD=abcd@123 REGISTRY_USER=no-user REGISTRY_PASSWORD=no-user-password +# ---- telemetry ------------------------------------------------------------- +# The three adapters ship metrics, traces and logs over OTLP/gRPC to the +# collector named here. hyperdx is ClickStack, on the observability profile, +# which `make up` starts and `make up-core` does not. +# +# ONE SWITCH, THREE SIGNALS. With OTEL_ENABLED=false the plugin builds no +# exporter and never dials -- so it cannot log a refused connection every few +# seconds. Set it false for `make up-core`, or for any stack whose collector +# is not running; leaving it true against a missing collector is the noisy +# case, not a broken one. +# +# Changing any of these means re-running bin/setup.py and recreating the +# adapters: they are rendered into the configs, not read at runtime. +OTEL_ENABLED=true +OTLP_ENDPOINT=hyperdx:4317 +OTEL_ENVIRONMENT=dev + # ---- discovery ------------------------------------------------------------- APP_NETWORK_ID=oan-dev BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 6249a62..8205a7f 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -376,7 +376,14 @@ def render(identities): (f"__{prefix}_ENCR_PRIVATE__", identity["encrPrivate"]), (f"__{prefix}_ENCR_PUBLIC__", identity["encrPublic"]), ("__PROVIDER_BINDING_KEY__", binding), - ("__MANDI_BINDING_KEY__", mandi_binding)): + ("__MANDI_BINDING_KEY__", mandi_binding), + # Telemetry. One switch drives all three signals: with every + # one false the plugin builds no exporter and never dials, so + # a stack running without the observability profile stays + # quiet instead of logging a refused connection on a loop. + ("__OTEL_ENABLED__", env("OTEL_ENABLED", "true")), + ("__OTLP_ENDPOINT__", env("OTLP_ENDPOINT", "hyperdx:4317")), + ("__OTEL_ENVIRONMENT__", env("OTEL_ENVIRONMENT", "dev"))): template = template.replace(placeholder, value) if "__" in template: sys.exit(f"setup: {role}.yaml still has unrendered placeholders") diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/docker-deployment/config/adapters/exp.yaml.tmpl index 94e0c21..a8be427 100644 --- a/docker-deployment/config/adapters/exp.yaml.tmpl +++ b/docker-deployment/config/adapters/exp.yaml.tmpl @@ -22,6 +22,42 @@ http: pluginManager: root: ./plugins +# --------------------------------------------------------------------------- +# OpenTelemetry. An application-level plugin rather than a module one: it runs +# its own background lifecycle -- an exporter and a periodic flush -- instead +# of being loaded per request by a step. +# +# It ships metrics, traces and logs over OTLP/gRPC to the collector named +# below. In this stack that is ClickStack, on the observability profile, which +# `make up` starts as step 5 and `make up-core` deliberately does not. +# +# WHICH IS WHY THE ENABLE FLAGS ARE A SWITCH. With all three false the plugin +# builds no exporter at all and returns a no-op provider -- it does not dial, +# so it cannot log a connection failure every few seconds. Point a stack with +# no collector at one and that is exactly what you get, so OTEL_ENABLED=false +# is the right setting for `make up-core`. +# +# serviceVersion is deliberately absent: left empty the plugin fills in the +# adapter's own build version, which is truer than anything written here and +# does not go stale. +# +# Not set, and worth knowing they exist: +# auditFieldsConfig a YAML file of masking rules and field selection for +# audit logs -- payloads are emitted whole without it. +# networkMetricsGranularity / networkMetricsFrequency +# network-level metric windows. +# timeInterval metric export period in seconds; defaults to 5. +plugins: + otelsetup: + id: otelsetup + config: + serviceName: "oan-exp-adapter" + environment: "__OTEL_ENVIRONMENT__" + otlpEndpoint: "__OTLP_ENDPOINT__" + enableMetrics: "__OTEL_ENABLED__" + enableTracing: "__OTEL_ENABLED__" + enableLogs: "__OTEL_ENABLED__" + modules: - name: exp-adapter # A subtree: every action lands here and the payload says which one it is. diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl index 2086c7e..568606a 100644 --- a/docker-deployment/config/adapters/network.yaml.tmpl +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -20,6 +20,42 @@ http: pluginManager: root: ./plugins +# --------------------------------------------------------------------------- +# OpenTelemetry. An application-level plugin rather than a module one: it runs +# its own background lifecycle -- an exporter and a periodic flush -- instead +# of being loaded per request by a step. +# +# It ships metrics, traces and logs over OTLP/gRPC to the collector named +# below. In this stack that is ClickStack, on the observability profile, which +# `make up` starts as step 5 and `make up-core` deliberately does not. +# +# WHICH IS WHY THE ENABLE FLAGS ARE A SWITCH. With all three false the plugin +# builds no exporter at all and returns a no-op provider -- it does not dial, +# so it cannot log a connection failure every few seconds. Point a stack with +# no collector at one and that is exactly what you get, so OTEL_ENABLED=false +# is the right setting for `make up-core`. +# +# serviceVersion is deliberately absent: left empty the plugin fills in the +# adapter's own build version, which is truer than anything written here and +# does not go stale. +# +# Not set, and worth knowing they exist: +# auditFieldsConfig a YAML file of masking rules and field selection for +# audit logs -- payloads are emitted whole without it. +# networkMetricsGranularity / networkMetricsFrequency +# network-level metric windows. +# timeInterval metric export period in seconds; defaults to 5. +plugins: + otelsetup: + id: otelsetup + config: + serviceName: "oan-network-adapter" + environment: "__OTEL_ENVIRONMENT__" + otlpEndpoint: "__OTLP_ENDPOINT__" + enableMetrics: "__OTEL_ENABLED__" + enableTracing: "__OTEL_ENABLED__" + enableLogs: "__OTEL_ENABLED__" + modules: - name: network-adapter # A subtree: every action lands here and the payload says which one it is. diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index 8ffcf06..be5a928 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -21,6 +21,42 @@ http: pluginManager: root: ./plugins +# --------------------------------------------------------------------------- +# OpenTelemetry. An application-level plugin rather than a module one: it runs +# its own background lifecycle -- an exporter and a periodic flush -- instead +# of being loaded per request by a step. +# +# It ships metrics, traces and logs over OTLP/gRPC to the collector named +# below. In this stack that is ClickStack, on the observability profile, which +# `make up` starts as step 5 and `make up-core` deliberately does not. +# +# WHICH IS WHY THE ENABLE FLAGS ARE A SWITCH. With all three false the plugin +# builds no exporter at all and returns a no-op provider -- it does not dial, +# so it cannot log a connection failure every few seconds. Point a stack with +# no collector at one and that is exactly what you get, so OTEL_ENABLED=false +# is the right setting for `make up-core`. +# +# serviceVersion is deliberately absent: left empty the plugin fills in the +# adapter's own build version, which is truer than anything written here and +# does not go stale. +# +# Not set, and worth knowing they exist: +# auditFieldsConfig a YAML file of masking rules and field selection for +# audit logs -- payloads are emitted whole without it. +# networkMetricsGranularity / networkMetricsFrequency +# network-level metric windows. +# timeInterval metric export period in seconds; defaults to 5. +plugins: + otelsetup: + id: otelsetup + config: + serviceName: "oan-provider-adapter" + environment: "__OTEL_ENVIRONMENT__" + otlpEndpoint: "__OTLP_ENDPOINT__" + enableMetrics: "__OTEL_ENABLED__" + enableTracing: "__OTEL_ENABLED__" + enableLogs: "__OTEL_ENABLED__" + modules: - name: oanProvider # A subtree, not one action. Without the trailing slash Go's ServeMux From 453d2246ca7a725d9801f3a091dd92021efdaf80 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 01:29:22 +0530 Subject: [PATCH 33/81] refactor: mount the Beckn surface at the root, not under /oan/ [OpenAgriNet/network-adapter#4] The /oan/ prefix earned nothing. Routing to an adapter is a gateway rule pointing at a container name and port, so the path never disambiguated anything -- it was just a segment every caller had to remember and every proxy rule had to carry. Actions are now served at the root: /select, /discover, /publish. The registry's baseUrl loses the suffix with it, so a peer calling /select works without the participant record naming a path. THE PROVIDER ADAPTER IS THE INTERESTING CASE, because it mounts two modules and they cannot both be "/" -- registering the same pattern twice panics at startup. The Beckn surface takes the subtree at "/" and publish takes the exact path /publish; Go's mux prefers the exact pattern for /publish and falls back to "/" for everything else. That has a consequence worth reading routing-provider.yaml for. The adapter takes the action from the URL, not the payload: it strips the module's mount path off the request path and matches what is left. Under "/" that leaves "select". Under the exact /publish it leaves the empty string -- so the rule keys on "" and, with no action left to append to a target, sets excludeAction and spells the target out in full. Confirmed the hard way first: mounting publish at /publish without that gave failed to determine route: endpoint '' is not supported for version 2.0.0 The alternative was a subtree of its own, posting to /internal/publish. Keeping the URL the provider's catalogue system already uses was worth more than a tidier routing file, and it also leaves the gateway's `location = /publish` deny matching exactly as before. Two comments were wrong and are fixed rather than moved: the provider template claimed the action "comes from the payload, never the URL", which is backwards. The validator is the only part that reads context.action, and it ignores the path it is handed -- so nothing reconciles the two. A mismatch does get caught in practice, but by the body shapes differing: POST /select carrying action "discover" is rejected SCH_FIELD_NOT_ALLOWED, verified. Tested end to end in oan-local, which now runs the same arrangement: publish, discover and select for both capabilities, 39 assertions, no failures. --- docker-deployment/README.md | 45 +++++++++++------ docker-deployment/bin/setup.py | 2 +- .../config/adapters/exp.yaml.tmpl | 2 +- .../config/adapters/network.yaml.tmpl | 2 +- .../config/adapters/provider.yaml.tmpl | 50 +++++++++++++------ .../config/adapters/routing-exp.yaml | 4 +- .../config/adapters/routing-network.yaml | 5 +- .../config/adapters/routing-provider.yaml | 26 ++++++++-- .../gateway/npm-custom/server_proxy.conf | 2 +- .../OAN-dev-flow.postman_collection.json | 12 ++--- 10 files changed, 100 insertions(+), 50 deletions(-) diff --git a/docker-deployment/README.md b/docker-deployment/README.md index fab93e9..303da06 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -114,10 +114,10 @@ in sixty days. hardening — it is the one thing standing between the public internet and an unauthenticated write into the catalogue. -The provider adapter mounts two modules: `/oan/` verifies the sender's -signature against the registry, but `oanProviderPublish` is mounted at `/` with -**no signature check at all**, because its intended caller is the provider's -own catalogue system inside the trust boundary. A proxy host pointed at +The provider adapter mounts two modules. The one at `/` verifies the sender's +signature against the registry; `oanProviderPublish`, on the exact path +`/publish`, has **no signature check at all**, because its intended caller is +the provider's own catalogue system inside the trust boundary. A proxy host pointed at `provider-adapter:9200` therefore exposes `/publish` to anyone. NPM's UI offers no way to route a host while withholding one path, so the block lives in `config/gateway/npm-custom/server_proxy.conf`, which NPM includes in **every** @@ -430,7 +430,7 @@ And through the gateway, once the proxy hosts exist: ```sh # the routed surface curl -s -o /dev/null -w '%{http_code}\n' \ - https://exp.oan.example.com/oan/search # reaches the adapter + https://exp.oan.example.com/search # reaches the adapter # the two that matter more curl -s -o /dev/null -w '%{http_code}\n' \ @@ -556,7 +556,7 @@ The rest of this section is one of those requests as curl, if you would rather see it than run it. ```sh -curl -s -X POST http://127.0.0.1:9202/oan/select \ +curl -s -X POST http://127.0.0.1:9202/select \ -H 'Content-Type: application/json' \ -d '{ "context": { @@ -671,10 +671,18 @@ in the registry by that key. So one adapter fronts both capabilities, and a third is a plugin plus two registry rows rather than a new route or a new port. -Each adapter's Beckn surface is one subtree, `/oan/`, and the payload's -`action` says which action it is. That is the path the registry publishes as -a participant's `baseUrl`, so a peer calling `/select` lands on the -module that answers select. +Each adapter's Beckn surface is mounted at the root, so a peer calling +`/select` lands on the module that answers select and the `baseUrl` +the registry publishes needs no path on it. There is no prefix to strip in a +gateway rule either: a proxy host forwards to a container name and port, and +the path arrives unchanged. + +**The action comes from the URL, not the payload.** The adapter strips the +module's mount path off the request path and matches what is left — `select`, +`discover` — against the routing config. The schema validator is the exception: +it reads `context.action` out of the body and ignores the path. Nothing +reconciles the two, though a mismatch usually fails validation anyway, since +two actions rarely accept the same body. Publishing enters at the **provider** adapter, which signs and forwards: @@ -685,11 +693,18 @@ curl -s -X POST http://127.0.0.1:9200/publish \ Three things about that: -- **`/publish` sits outside `/oan/`, at the root.** It is not part of this - adapter's Beckn surface — it arrives from inside the provider's own - deployment — so it must not shadow it. Go's mux takes the longest matching - pattern, so `/oan/select` still reaches the capability module and only - `/publish` falls to the root one. +- **It is mounted on the exact path `/publish`,** while the Beckn surface + takes the whole subtree at `/`. Go's mux prefers the exact pattern for + `/publish` and falls back to `/` for everything else, so `/select` still + reaches the capability module. The two can coexist only because the patterns + differ — give both the same path and registration panics at startup. +- **Which is why `routing-provider.yaml` keys on an empty endpoint.** Stripping + the mount path `/publish` off the request path `/publish` leaves nothing, so + the empty string *is* the endpoint, and there is no action left for the + router to append to a target. Hence `excludeAction: true` and a target URL + written out in full. It looks odd; the alternative was posting to something + like `/internal/publish` instead, and keeping the URL the provider's + catalogue system already uses was worth more. - **It is a second module, and has to be.** The routing step fails any action missing from its config, so routing publish from the module that answers select would mean listing select too — and listing select would proxy it to diff --git a/docker-deployment/bin/setup.py b/docker-deployment/bin/setup.py index 8205a7f..859bf68 100755 --- a/docker-deployment/bin/setup.py +++ b/docker-deployment/bin/setup.py @@ -229,7 +229,7 @@ def node(participant_id, name, role, public_key): resolved here: routing between the adapters is the router plugin's config, which uses the compose service names.""" return {"participantId": participant_id, "name": name, "type": "node", - "status": "active", "baseUrl": f"https://{participant_id}/oan", + "status": "active", "baseUrl": f"https://{participant_id}", "role": role, "keys": signing_key_block(public_key)} diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/docker-deployment/config/adapters/exp.yaml.tmpl index a8be427..3b5e645 100644 --- a/docker-deployment/config/adapters/exp.yaml.tmpl +++ b/docker-deployment/config/adapters/exp.yaml.tmpl @@ -61,7 +61,7 @@ plugins: modules: - name: exp-adapter # A subtree: every action lands here and the payload says which one it is. - path: /oan/ + path: / handler: type: std role: bap diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl index 568606a..89f3700 100644 --- a/docker-deployment/config/adapters/network.yaml.tmpl +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -59,7 +59,7 @@ plugins: modules: - name: network-adapter # A subtree: every action lands here and the payload says which one it is. - path: /oan/ + path: / handler: type: std # bpp because this adapter RECEIVES rather than originates. The role diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index be5a928..e1366b2 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -1,6 +1,7 @@ # OAN provider adapter -- dev deployment. # -# Serves /oan/ synchronously: verifies the sender against the registry, +# Serves the Beckn actions at the root synchronously: verifies the sender +# against the registry, # resolves the capability's call plan, calls the provider, and answers with the # mapped result. No callback -- the answer is the HTTP response. appName: "oan-provider-adapter" @@ -59,10 +60,22 @@ plugins: modules: - name: oanProvider - # A subtree, not one action. Without the trailing slash Go's ServeMux - # matches exactly, so /oan/select would mount that action and 404 the - # rest. Which action it is comes from the payload, never the URL. - path: /oan/ + # A subtree, not one action. "/" with the trailing slash is a prefix + # match, so every action lands here; an exact pattern like "/select" + # would mount that one action and 404 the rest. + # + # Which action it is comes from the URL, not the payload: the mount path + # is stripped off the request path and what remains -- "select", + # "discover" -- is what the routing config matches on. + # + # The schema validator is handed that same stripped path and ignores it, + # reading context.action out of the payload instead. Nothing reconciles + # the two. A mismatch is caught anyway in practice -- POST /select + # carrying action "discover" is validated against the discover schema and + # rejected with SCH_FIELD_NOT_ALLOWED -- but it is the two body shapes + # differing that catches it, not a check that the URL and the action + # agree. + path: / handler: type: std role: bpp @@ -173,17 +186,22 @@ modules: # The outbound leg: the provider's own catalogue system publishing to the # network layer. POST /publish. # - # Mounted at the root, and /oan/ above wins for anything under it because - # Go's mux takes the longest matching pattern. That split is deliberate: - # /oan/ is this adapter's Beckn surface, the path the registry publishes as - # its baseUrl, so a network peer calling /select has to land on the - # module that answers select. Publishing is not part of that surface -- it - # arrives from inside this provider's own deployment -- so it sits outside - # it rather than shadowing it. + # Mounted on the EXACT path /publish while the module above takes the whole + # subtree at /. Go's mux prefers the exact pattern for /publish and falls + # back to / for everything else, so /select still reaches the module that + # answers it. The two can coexist only because the patterns differ -- give + # both the same path and registration panics at startup. + # + # THIS IS WHY routing-provider.yaml LOOKS ODD. The adapter takes the action + # from the URL, not from the payload: it strips the module's mount path off + # the request path and matches whatever is left. Under / that leaves + # "select"; under the exact /publish it leaves the empty string. So the + # routing rule keys on "" and spells its target out in full with + # excludeAction, because there is no action left to append. # - # It has to be a second module, not another action on /oan/: the routing - # step fails any action missing from its config, so routing publish from - # the module that answers select would mean listing select too -- and + # It has to be a second module, not another action on the one above: the + # routing step fails any action missing from its config, so routing publish + # from the module that answers select would mean listing select too -- and # listing select would proxy it to the network layer instead of answering # it here, which is the one thing that module does. # @@ -196,7 +214,7 @@ modules: # Authorization header's keyId, taken from keyManager below, and that is # what the network layer verifies against the registry. - name: oanProviderPublish - path: / + path: /publish handler: type: std # bap because this module SENDS. diff --git a/docker-deployment/config/adapters/routing-exp.yaml b/docker-deployment/config/adapters/routing-exp.yaml index 1a9c2f9..e09d092 100644 --- a/docker-deployment/config/adapters/routing-exp.yaml +++ b/docker-deployment/config/adapters/routing-exp.yaml @@ -10,14 +10,14 @@ routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://network-adapter:9201/oan" + url: "http://network-adapter:9201" endpoints: - discover - version: "2.0.0" targetType: "url" target: - url: "http://provider-adapter:9200/oan" + url: "http://provider-adapter:9200" endpoints: - select - init diff --git a/docker-deployment/config/adapters/routing-network.yaml b/docker-deployment/config/adapters/routing-network.yaml index 41b0723..257f7cd 100644 --- a/docker-deployment/config/adapters/routing-network.yaml +++ b/docker-deployment/config/adapters/routing-network.yaml @@ -3,8 +3,9 @@ # One job: hand the catalogue actions to the discovery service, which is a # service in this same compose and so reachable by name. # -# The service serves /discover and /publish at its root, so no /oan prefix -# here: targetType "url" appends the action to whatever base is given. +# The service serves /discover and /publish at its root, and so does this +# adapter now: targetType "url" appends the action to whatever base is +# given, so the base is the bare host and port. # # Both actions are one rule because they share a target. publish arrives from # the provider adapter, discover from the experience adapter, and neither is diff --git a/docker-deployment/config/adapters/routing-provider.yaml b/docker-deployment/config/adapters/routing-provider.yaml index bb9dd0d..7fdae7c 100644 --- a/docker-deployment/config/adapters/routing-provider.yaml +++ b/docker-deployment/config/adapters/routing-provider.yaml @@ -1,15 +1,31 @@ -# Provider adapter routing -- for the root module only. +# Provider adapter routing -- for the publish module only. # # Publishing is the one thing this adapter sends rather than answers. The # provider's own catalogue system posts POST /publish here, and the catalogue # has to reach the network layer, which is what fronts the discovery service. # -# targetType "url" appends the action to the base, so this becomes -# http://network-adapter:9201/oan/publish. +# WHY THE ENDPOINT IS EMPTY, AND WHY THE TARGET IS SPELT OUT IN FULL. +# +# The adapter derives the action from the URL, not from the payload: it strips +# the module's mount path off the front of the request path and whatever is +# left is the endpoint it matches here. Every other module is mounted on a +# subtree -- "/" -- so /select leaves "select" and /discover leaves +# "discover". +# +# This module is mounted on the exact path /publish, because "/" already +# belongs to the module that answers select and two modules cannot share a +# pattern. Stripping /publish off /publish leaves the empty string, so the +# empty string is the endpoint, and there is no action left to append to a +# target. Hence excludeAction and the full URL. +# +# The alternative was mounting this on a subtree of its own and posting to +# something like /internal/publish. This keeps the URL the provider's +# catalogue system already posts to. routingRules: - version: "2.0.0" targetType: "url" target: - url: "http://network-adapter:9201/oan" + url: "http://network-adapter:9201/publish" + excludeAction: true endpoints: - - publish + - "" diff --git a/docker-deployment/config/gateway/npm-custom/server_proxy.conf b/docker-deployment/config/gateway/npm-custom/server_proxy.conf index 2399014..cc5d76d 100644 --- a/docker-deployment/config/gateway/npm-custom/server_proxy.conf +++ b/docker-deployment/config/gateway/npm-custom/server_proxy.conf @@ -5,7 +5,7 @@ # belong here. There is exactly one, and it matters more than the rest of this # stack's edge configuration combined. -# The provider adapter mounts TWO modules. /oan/ is its Beckn surface and +# The provider adapter mounts TWO modules. / is its Beckn surface and # verifies the sender's signature against the registry. But oanProviderPublish # is mounted at `/` with NO validateSign at all, because its intended caller is # the provider's own catalogue system, inside the trust boundary -- the same diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index 9d63533..b984826 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -125,9 +125,9 @@ } ], "url": { - "raw": "{{expAdapterUrl}}/oan/discover", + "raw": "{{expAdapterUrl}}/discover", "host": [ - "{{expAdapterUrl}}/oan/discover" + "{{expAdapterUrl}}/discover" ] }, "body": { @@ -166,7 +166,7 @@ "value": "application/json" } ], - "url": "{{expAdapterUrl}}/oan/discover", + "url": "{{expAdapterUrl}}/discover", "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing.", "body": { "mode": "raw", @@ -218,9 +218,9 @@ } ], "url": { - "raw": "{{expAdapterUrl}}/oan/select", + "raw": "{{expAdapterUrl}}/select", "host": [ - "{{expAdapterUrl}}/oan/select" + "{{expAdapterUrl}}/select" ] }, "body": { @@ -282,7 +282,7 @@ "value": "application/json" } ], - "url": "{{expAdapterUrl}}/oan/select", + "url": "{{expAdapterUrl}}/select", "description": "The same endpoint as request 5, the same adapter, a different capability. Nothing routes this: the payload's provider id and resourceAttributes @type form a binding key, the mandi step recognises it and the weather step passes it through.\n\nThe answer is a Direct openagrinet:MandiPrice per price record. Its prices arrive from the upstream as STRINGS with Title Case keys containing spaces, so the mapping converts them; and a record that reported no minimum or maximum must come back with those absent rather than zeroed, which is what the last assertion checks.\n\nThe upstream's credential is a query parameter, which the adapter adds from an environment variable and redacts from the URL it logs.", "body": { "mode": "raw", From 19d481ecea717857c43aad9e6773d4623cc7ccc9 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:37:04 +0530 Subject: [PATCH 34/81] feat: add a restart target for the app tier [#4] Restarts registry, discovery and the three adapters -- the services holding OAN's own code and config -- and leaves keycloak, both databases and the edge running. Excluded deliberately rather than for brevity. The databases are state, and keycloak reads its realm from one that was seeded on first boot rather than from a file, so restarting them to pick up a config change is downtime that cannot have helped. NPM keeps its routing table in SQLite, which a restart does not re-read either; restart-edge stays the separate target for the one case that needs it. Uses `docker compose restart` rather than `up -d --force-recreate`: a restart keeps each container and therefore its address, so NPM's cached proxy_pass targets stay valid. A recreate changes the address and leaves every proxy host 502ing until restart-edge runs. --- docker-deployment/Makefile | 6 +++++- docker-deployment/bin/stack.sh | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docker-deployment/Makefile b/docker-deployment/Makefile index 4a3b4ab..1b57500 100644 --- a/docker-deployment/Makefile +++ b/docker-deployment/Makefile @@ -13,7 +13,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core down destroy setup gateway observability restart-edge ps logs +.PHONY: help up up-core down destroy setup gateway observability restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -38,6 +38,10 @@ observability: ; @$(STACK) observability # ------------------------------------------------------------------ the rest setup: ; @$(STACK) setup + +# The app tier only -- registry, discovery, the three adapters. Not keycloak, +# not the databases, not the edge. +restart: ; @$(STACK) restart restart-edge: ; @$(STACK) restart-edge ps: ; @$(STACK) ps diff --git a/docker-deployment/bin/stack.sh b/docker-deployment/bin/stack.sh index 5f3e63d..06343f4 100755 --- a/docker-deployment/bin/stack.sh +++ b/docker-deployment/bin/stack.sh @@ -230,6 +230,37 @@ observability() { info "UI on 127.0.0.1:8085. There is no login in front of it -- keep it on loopback." } +# --------------------------------------------------------------- restart + +# The services that hold OAN's own code and config, and nothing else. +# +# Deliberately excludes the two Postgres instances and keycloak: those are +# state, they are slow to come back, and nothing you change in this repo +# alters their behaviour -- keycloak reads its realm from a database that was +# seeded on first boot, not from a file you can edit. Restarting them to pick +# up a config change is a minute of downtime that cannot have helped. +# +# It also excludes nginx-proxy-manager, whose routing table lives in a SQLite +# database rather than in anything a restart would re-read. `restart-edge` is +# the separate target for the one case that does need it. +APP_SERVICES=(registry discovery provider-adapter network-adapter exp-adapter) + +# `restart`, not `up -d --force-recreate`. A restart keeps the container and +# therefore its address, so NPM's cached proxy_pass targets stay valid -- a +# recreate changes the address and leaves every proxy host 502ing until +# `restart-edge` runs. Same reason the compose file spells this out. +# +# What this picks up: the registry re-reads config/registry/schemas, and each +# adapter re-reads the config setup.py rendered for it. What it does not pick +# up is a changed image or a changed environment, both of which need the +# container recreated -- use `make up` for those. +restart_app() { + step 1 1 "restarting registry, discovery and the three adapters" + info "keycloak, both databases and the edge are left alone" + docker compose restart "${APP_SERVICES[@]}" + docker compose ps --format 'table {{.Service}}\t{{.Status}}' +} + # ----------------------------------------------------------------- misc # NPM writes a literal proxy_pass hostname per proxy host, which nginx resolves @@ -262,6 +293,7 @@ bin/stack.sh setup re-run bin/setup.py only gateway start nginx-proxy-manager on its own (public, 80/443) observability start hyperdx on its own + restart restart registry, discovery and the adapters only restart-edge restart NPM after recreating an adapter (fixes a 502) ps docker compose ps logs [service] docker compose logs -f @@ -277,6 +309,7 @@ case "${1:-}" in setup) setup ;; gateway) gateway ;; observability) observability ;; + restart) restart_app ;; restart-edge) restart_edge ;; ps) docker compose "${PROFILES[@]}" ps ;; logs) shift; docker compose "${PROFILES[@]}" logs -f "$@" ;; From feab11733b193605980c8465ab231523c0850714 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:52:10 +0530 Subject: [PATCH 35/81] fix: probe a route the hyperdx image actually serves [#4] Port 8080 is a Next.js app -- the HyperDX UI -- and has no /health route, so the check 404ed. The previous fix to this healthcheck corrected the address it dialled and stopped there; two faults were stacked, and repairing the connection only revealed that the path had never existed. "/" is what the published 8085 serves and what proves the container is doing its job, so it is both the working probe and the honest one. --- docker-deployment/docker-compose.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index 1f89a58..c6db409 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -483,7 +483,12 @@ services: # on 8085 serves perfectly well. An unhealthy container that is in fact # working is worse than either state on its own, because it sends you # looking at ClickHouse. - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/health"] + # + # And "/" rather than "/health", which this image does not serve: 8080 is + # a Next.js app -- the HyperDX UI -- and has no such route, so the probe + # 404ed even once it could connect. Two faults, and fixing the first only + # revealed the second. + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/"] interval: 10s timeout: 5s # ClickHouse creates its system tables on a cold volume, which takes From 6c123d3acb2e351e4fd20d53e8840ae485a4ffdf Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:09:37 +0530 Subject: [PATCH 36/81] fix: send the hyperdx healthcheck a GET, and check the collector too [#4] `--spider` sends HEAD, and Next.js answers HEAD / with a 404 while serving GET / with a 200 -- so the probe reported a broken UI that was working throughout. Dropping the flag makes it a real GET with the body discarded. Also checks 13133, the collector's health_check extension. This container is two things: 8080 is the UI that 8085 publishes, and the collector is what receives telemetry on 4317/4318. The collector can die while the UI keeps serving, which loses data silently -- so the check that only looked at the web page would have reported healthy through exactly the failure worth catching. --- docker-deployment/docker-compose.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index c6db409..ff40d10 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -475,20 +475,19 @@ services: volumes: - hyperdx-data:/var/lib/clickhouse healthcheck: - # wget, not curl: this image has busybox. + # wget, not curl: this image has busybox. And a GET, not `--spider`, + # which sends HEAD -- Next.js answers HEAD / with a 404 while serving + # GET / perfectly, so the probe failed against a UI that was working. # - # 127.0.0.1 and not localhost, which is not a style preference. busybox - # resolves localhost to ::1 first, and this image binds v4 only -- so the - # check fails with "Connection refused" against [::1]:8080 while the UI - # on 8085 serves perfectly well. An unhealthy container that is in fact - # working is worse than either state on its own, because it sends you - # looking at ClickHouse. + # 127.0.0.1 rather than localhost, because busybox resolves localhost to + # ::1 first and these listeners are v4. # - # And "/" rather than "/health", which this image does not serve: 8080 is - # a Next.js app -- the HyperDX UI -- and has no such route, so the probe - # 404ed even once it could connect. Two faults, and fixing the first only - # revealed the second. - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/"] + # Two checks, because this container is two things. 8080 is the Next.js + # UI -- what 8085 publishes and what you actually open. 13133 is the + # collector's health_check extension, and the collector is what receives + # telemetry on 4317/4318: it can die while the UI keeps serving, which + # loses data silently and is the failure worth catching. + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/ && wget -q -O /dev/null http://127.0.0.1:13133/"] interval: 10s timeout: 5s # ClickHouse creates its system tables on a cold volume, which takes From 95b456cb2b7098598fb076320686cc74456e1adc Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:42:30 +0530 Subject: [PATCH 37/81] feat: open /publish on the edge for catalogue testing [#4] Comments out the global deny, so POST /publish now reaches the provider adapter through NPM rather than returning 403. What this exposes, stated plainly because the comment above it argues the other way: oanProviderPublish mounts at /publish with no validateSign -- the module at / verifies the sender's signature against the registry, this one deliberately does not, because its intended caller is the provider's own catalogue system inside the trust boundary. provider-adapter sits on oan-edge, so with the deny gone anyone who reaches the hostname can write into the catalogue with no token, no signature and no address allowlist. Deliberate, for testing the publish flow end to end against the deployed stack. The deny is one uncomment away and the block above it records the durable options -- a tunnel, or reaching provider-adapter from inside the VPC -- for when this stops being a dev box. --- .../config/gateway/npm-custom/server_proxy.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-deployment/config/gateway/npm-custom/server_proxy.conf b/docker-deployment/config/gateway/npm-custom/server_proxy.conf index cc5d76d..85d0a65 100644 --- a/docker-deployment/config/gateway/npm-custom/server_proxy.conf +++ b/docker-deployment/config/gateway/npm-custom/server_proxy.conf @@ -25,6 +25,6 @@ # provider-adapter directly -- an endpoint with no credential to check does not # belong on the public edge, and an address allowlist in front of it is a # statement about the network, which is where it should be made. -location = /publish { - deny all; -} +# location = /publish { +# deny all; +# } From 24a3c247c568b2c4674e194bcf3893abde961e49 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:47:35 +0530 Subject: [PATCH 38/81] feat: add a pull target that survives NPM's chown [#4] config/gateway/npm-custom is bind-mounted into NPM, and NPM's s6 init chowns everything under /data/nginx on every start. The files end up owned by a UID that is not the operator, and git cannot unlink them to update: error: unable to unlink old '.../server_proxy.conf': Permission denied The mount cannot be :ro -- that stops nginx from starting at all -- and git does not track ownership, so there is no fix that holds. Taking the files back first is the whole workaround, and it belongs in a target rather than in someone's memory of having hit it before. Only chowns when something is actually owned by someone else, so the sudo prompt appears when it is needed and not otherwise. --- docker-deployment/Makefile | 5 ++++- docker-deployment/bin/stack.sh | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docker-deployment/Makefile b/docker-deployment/Makefile index 1b57500..9b06361 100644 --- a/docker-deployment/Makefile +++ b/docker-deployment/Makefile @@ -13,7 +13,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core down destroy setup gateway observability restart restart-edge ps logs +.PHONY: help up up-core down destroy setup gateway observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -39,6 +39,9 @@ observability: ; @$(STACK) observability setup: ; @$(STACK) setup +# NPM chowns the files it is given, so a plain `git pull` fails on them. +pull: ; @$(STACK) pull + # The app tier only -- registry, discovery, the three adapters. Not keycloak, # not the databases, not the edge. restart: ; @$(STACK) restart diff --git a/docker-deployment/bin/stack.sh b/docker-deployment/bin/stack.sh index 06343f4..7e5b1e0 100755 --- a/docker-deployment/bin/stack.sh +++ b/docker-deployment/bin/stack.sh @@ -230,6 +230,42 @@ observability() { info "UI on 127.0.0.1:8085. There is no login in front of it -- keep it on loopback." } +# ------------------------------------------------------------------ pull + +# `git pull`, with the one thing that otherwise stops it. +# +# config/gateway/npm-custom is bind-mounted into NPM, and NPM's s6 init chowns +# everything under /data/nginx on every start -- so those two files end up +# owned by a UID that is not you, and git cannot unlink them to update: +# +# error: unable to unlink old '.../server_proxy.conf': Permission denied +# +# The mount cannot be :ro (that stops nginx from starting at all -- see the +# comment on the mount), and git does not track ownership, so there is nothing +# to fix once and for all. Taking the files back before pulling is the whole +# workaround, and it belongs in a target rather than in someone's memory. +pull() { + step 1 2 "taking back ownership of config/gateway/npm-custom" + if [ -n "$(find config/gateway/npm-custom ! -user "$(id -un)" -print -quit 2>/dev/null)" ]; then + sudo chown -R "$(id -un):$(id -gn)" config/gateway/npm-custom + info "done -- NPM had chowned them on its last start" + else + info "already yours, nothing to do" + fi + + step 2 2 "git pull" + git -C "$(git rev-parse --show-toplevel)" pull + cat <<'NEXT' + + Then apply what came in: + + make up new services, changed images or .env + make restart changed adapter or registry config + make restart-edge changed config/gateway/npm-custom + +NEXT +} + # --------------------------------------------------------------- restart # The services that hold OAN's own code and config, and nothing else. @@ -293,6 +329,7 @@ bin/stack.sh setup re-run bin/setup.py only gateway start nginx-proxy-manager on its own (public, 80/443) observability start hyperdx on its own + pull git pull, fixing the npm-custom ownership first restart restart registry, discovery and the adapters only restart-edge restart NPM after recreating an adapter (fixes a 502) ps docker compose ps @@ -309,6 +346,7 @@ case "${1:-}" in setup) setup ;; gateway) gateway ;; observability) observability ;; + pull) pull ;; restart) restart_app ;; restart-edge) restart_edge ;; ps) docker compose "${PROFILES[@]}" ps ;; From 94c14061846464d5986d11f4026187c29545fc1a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 12:52:31 +0530 Subject: [PATCH 39/81] feat: ship the registry requests and an environment file with the collection [OpenAgriNet/network-adapter#4] The collection had publish, discover and select and nothing else, on the reasoning that the registry has no public route so a Postman request could not reach it. That was true of the route and wrong as a conclusion: the registry is reachable over a tunnel, which is how anyone administers this stack, and the request bodies are the clearest documentation of what a registry record actually looks like. So it gains the whole set: a write token, the three adapter identities, both upstreams, both capability bindings, updates, and both searches. Nineteen requests. AND AN ENVIRONMENT FILE, which is the part that makes it portable. Every URL variable still defaults to loopback in the COLLECTION, so importing the collection alone works against a tunnel with nothing to fill in. The environment carries the same keys, and Postman resolves an environment variable ahead of a collection variable of the same name -- so pointing at a deployed experience or network adapter is six fields in the environment, with the loopback defaults left intact for the next person. No VM hostname or address is committed in either file. Those get shared separately, and the environment file is where they belong. A default run deliberately changes nothing, so it is safe against a live stack. The creates report "already present" -- the registry is append-only, so a second run cannot succeed -- and the test discriminates on the message rather than the status code, because a duplicate is a 500 while a bad body is a 400 and a missing token is a bare 401. Only a duplicate passes. The updates write back the same value the variable already holds. The updates were the interesting part to get right. PUT takes an OSID, not a participantId, because the registry addresses a record by the id it assigned on write -- so each PUT resolves that in a pre-request script and stays runnable on its own. And a partial body MERGES: sending only baseUrl leaves name, type and status alone, which is what makes repointing a mock URL a one-field request. Adds networkAdapterUrl as a variable no request uses. Discover reaches the network adapter through exp and publish through the provider adapter, so nothing here calls it directly -- but it is the other adapter a deployment exposes, and its /publish and /discover both verify signatures. Postman does not sign, so those calls are not scripted. Verified by running this collection against the oan-local stack with nothing but an environment file overriding the values: 22 requests, 50 assertions, no failures. That exercises the collection and the override mechanism at once. --- .../OAN-dev-flow.postman_collection.json | 957 +++++++++++++++++- .../OAN-dev.postman_environment.json | 200 ++++ .../postman-collection/README.md | 116 ++- 3 files changed, 1208 insertions(+), 65 deletions(-) create mode 100644 docker-deployment/postman-collection/OAN-dev.postman_environment.json diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index b984826..90727c4 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -1,44 +1,965 @@ { "info": { - "name": "OAN dev \u2014 publish, discover, select for two capabilities", - "description": "The whole flow against a docker-deployment stack: a publish, a discover and a select for each of the two capabilities.\n\nTHERE ARE NO REGISTRY REQUESTS HERE, and that is deliberate. The registry is not reachable from outside the stack -- no port beyond loopback, no proxy host in front of it -- so nothing in this collection could create or read a registry row. `bin/setup.py` seeds all of it: five participants and two capability bindings, from the same .env the adapter configs are rendered from, which is what keeps the two from disagreeing.\n\nSo the prerequisite is `make setup` followed by `make up`. After that everything here works with no value to fill in -- providerId and mandiProviderId are already the ids that deployment uses.\n\nRequests 5 and 6 are the point. They hit the same endpoint on the same adapter and different domain packages answer them, because each provider step recognises its own binding key from the payload and passes through anything else. Nothing routes by URL or by domain.\n\nRun in order the first time: 1 and 2 publish what 3 and 4 look for.\n\nThe ports are loopback, so from a workstation tunnel first:\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 you@the-vm", + "name": "OAN dev \u2014 registry, publish, discover, select", + "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nRUN IN ORDER THE FIRST TIME. Request 1 issues the token that 2-11 need. The publish requests seed the catalogues that discover searches for.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "variable": [ + { + "key": "registryUrl", + "value": "http://localhost:8081/api/v1", + "description": "Loopback. Tunnel to the VM if it is not this machine." + }, + { + "key": "keycloakUrl", + "value": "http://localhost:8080", + "description": "Issues the write token." + }, + { + "key": "keycloakRealm", + "value": "sunbird-rc" + }, + { + "key": "keycloakClientId", + "value": "registry-frontend" + }, + { + "key": "registryUser", + "value": "no-user" + }, + { + "key": "registryPassword", + "value": "no-user-password" + }, + { + "key": "token", + "value": "", + "description": "Set by request 1. Do not fill in by hand." + }, + { + "key": "targetOsid", + "value": "", + "description": "Set by the PUT requests' pre-request scripts. Do not fill in by hand." + }, { "key": "expAdapterUrl", - "value": "http://127.0.0.1:9202", - "description": "The experience adapter -- the only one that takes an unsigned request." + "value": "http://localhost:9202", + "description": "Takes unsigned requests -- the app is inside the trust boundary." }, { "key": "providerAdapterUrl", - "value": "http://127.0.0.1:9200", - "description": "Where publish enters: the provider adapter signs it and forwards to the network layer." + "value": "http://localhost:9200", + "description": "Where publish enters." + }, + { + "key": "networkAdapterUrl", + "value": "http://localhost:9201", + "description": "The network layer adapter -- the peer-facing surface. No request in this collection uses it: discover reaches it via the experience adapter, and publish via the provider adapter. It is here because it is the one adapter besides exp that a deployment exposes publicly, and because its /publish and /discover both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not scripted here." + }, + { + "key": "discoveryUrl", + "value": "http://localhost:8090" + }, + { + "key": "expNodeId", + "value": "exp.oan.dev" + }, + { + "key": "expNodeKey", + "value": "", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "networkNodeId", + "value": "network.oan.dev" + }, + { + "key": "networkNodeKey", + "value": "", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "providerNodeId", + "value": "provider.oan.dev" + }, + { + "key": "providerNodeKey", + "value": "", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." }, { "key": "providerId", "value": "mausamgram-mock", - "description": "AS DEPLOYED: PROVIDER_PARTICIPANT_ID in .env. Half of the weather binding key." + "description": "The weather upstream. Half of its binding key." + }, + { + "key": "weatherCapability", + "value": "openagrinet:WeatherObservation" + }, + { + "key": "weatherBindingKey", + "value": "mausamgram-mock|openagrinet:WeatherObservation", + "description": "Used to look up the binding's osid for the update." + }, + { + "key": "weatherBaseUrl", + "value": "http://mockimd:9100", + "description": "A compose service name: reached from inside the network." + }, + { + "key": "weatherPath", + "value": "/get-daily" + }, + { + "key": "weatherMappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml" }, { "key": "mandiProviderId", "value": "agmarknet-mock", - "description": "AS DEPLOYED: MANDI_PARTICIPANT_ID in .env. Half of the mandi binding key." + "description": "The mandi upstream. Half of its binding key." + }, + { + "key": "mandiCapability", + "value": "openagrinet:MandiPrice" + }, + { + "key": "mandiBindingKey", + "value": "agmarknet-mock|openagrinet:MandiPrice" + }, + { + "key": "mandiBaseUrl", + "value": "http://mockagmarknet:9101" + }, + { + "key": "mandiPath", + "value": "/v1/fetch-agmarknet-vistaar" + }, + { + "key": "mandiMappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml" }, { "key": "networkId", - "value": "oan-dev", - "description": "APP_NETWORK_ID in .env. Discovery scopes by this: a catalogue published under a different one is invisible to the query." + "value": "oan-dev" }, { "key": "domain", - "value": "oan-dev", - "description": "Also required for a discover to match." + "value": "oan-dev" } ], "item": [ { - "name": "1. Publish \u2014 weather catalogue", + "name": "1. Registry \u2014 get a write token", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Forwarded-Host", + "value": "keycloak:8080" + }, + { + "key": "X-Forwarded-Proto", + "value": "http" + } + ], + "url": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", + "body": { + "mode": "urlencoded", + "urlencoded": [ + { + "key": "client_id", + "value": "{{keycloakClientId}}" + }, + { + "key": "grant_type", + "value": "password" + }, + { + "key": "username", + "value": "{{registryUser}}" + }, + { + "key": "password", + "value": "{{registryPassword}}" + } + ] + }, + "description": "Every registry WRITE needs this. Searches take no token at all.\n\nTHE TWO X-Forwarded-* HEADERS ARE NOT OPTIONAL, and keycloak:8080 is the container-internal address on purpose -- not whatever port Keycloak is published on. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and every write below returns 401 with an empty body.\n\nSaved to {{token}}, so run this first.\n\nA 500 here usually means Keycloak's realm is missing -- it shares a database with the registry, so wiping the registry volume takes the realm with it. Restarting Keycloak re-imports it." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const j = pm.response.json();", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", + "pm.collectionVariables.set(\"token\", j.access_token);" + ] + } + } + ] + }, + { + "name": "2. Registry \u2014 create the exp node", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{expNodeId}}\",\n \"name\": \"OAN experience layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{expNodeId}}\",\n \"role\": \"consumer\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{expNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Same as the other creates, with one extra allowance: this collection ships", + "// the node key blank on deployments where it is not knowable, and an empty key", + "// fails schema validation before the duplicate check is ever reached. That is", + "// reported rather than failed.", + "const key = pm.variables.get(\"expNodeKey\") || \"\";", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, already present, or awaiting a key\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " if (msg.includes(\"duplicate key\")) return; // seeded already", + " if (!key && msg.includes(\"does not match pattern\")) {", + " console.log(\"not attempted: expNodeKey is empty -- paste it from keys/keys.json\");", + " return;", + " }", + " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", + "});" + ] + } + } + ] + }, + { + "name": "3. Registry \u2014 create the network node", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{networkNodeId}}\",\n \"name\": \"OAN network layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{networkNodeId}}\",\n \"role\": \"network\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{networkNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Same as the other creates, with one extra allowance: this collection ships", + "// the node key blank on deployments where it is not knowable, and an empty key", + "// fails schema validation before the duplicate check is ever reached. That is", + "// reported rather than failed.", + "const key = pm.variables.get(\"networkNodeKey\") || \"\";", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, already present, or awaiting a key\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " if (msg.includes(\"duplicate key\")) return; // seeded already", + " if (!key && msg.includes(\"does not match pattern\")) {", + " console.log(\"not attempted: networkNodeKey is empty -- paste it from keys/keys.json\");", + " return;", + " }", + " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", + "});" + ] + } + } + ] + }, + { + "name": "4. Registry \u2014 create the provider node", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{providerNodeId}}\",\n \"name\": \"OAN provider layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{providerNodeId}}\",\n \"role\": \"provider\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{providerNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Same as the other creates, with one extra allowance: this collection ships", + "// the node key blank on deployments where it is not knowable, and an empty key", + "// fails schema validation before the duplicate check is ever reached. That is", + "// reported rather than failed.", + "const key = pm.variables.get(\"providerNodeKey\") || \"\";", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, already present, or awaiting a key\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " if (msg.includes(\"duplicate key\")) return; // seeded already", + " if (!key && msg.includes(\"does not match pattern\")) {", + " console.log(\"not attempted: providerNodeKey is empty -- paste it from keys/keys.json\");", + " return;", + " }", + " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", + "});" + ] + } + } + ] + }, + { + "name": "5. Registry \u2014 create the weather upstream", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } + } + ] + }, + { + "name": "6. Registry \u2014 create the weather capability binding", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema", + "body": { + "mode": "raw", + "raw": "{\n \"bindingKey\": \"{{providerId}}|{{weatherCapability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{weatherCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } + } + ] + }, + { + "name": "7. Registry \u2014 create the mandi upstream", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } + } + ] + }, + { + "name": "8. Registry \u2014 create the mandi capability binding", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema", + "body": { + "mode": "raw", + "raw": "{\n \"bindingKey\": \"{{mandiProviderId}}|{{mandiCapability}}\",\n \"participantId\": \"{{mandiProviderId}}\",\n \"capabilityCode\": \"{{mandiCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{mandiPath}}\",\n \"mappings\": \"{{mandiMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } + } + ] + }, + { + "name": "9. Registry \u2014 update the weather upstream URL (PUT)", + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant/{{targetOsid}}", + "body": { + "mode": "raw", + "raw": "{\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The registry addresses a record by the osid it assigned on write, not by", + "// participantId. So an update has to look the osid up first. Doing it here rather", + "// than in a preceding request keeps this one runnable on its own.", + "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", + "const want = pm.variables.replaceIn(\"{{providerId}}\");", + "const filters = {}; filters[\"participantId\"] = { eq: want };", + "", + "pm.sendRequest({", + " url: base + \"/Participant/search\",", + " method: \"POST\",", + " header: { \"Content-Type\": \"application/json\" },", + " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", + "}, function (err, res) {", + " if (err) { console.log(\"osid lookup failed: \" + err); return; }", + " const rows = ((res.json() || {}).data) || [];", + " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", + " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A partial body MERGES: fields you leave out keep their stored values. That is", + "// what makes this safe to re-run -- it writes the same value the variable already", + "// holds, so a default run changes nothing. Change the variable to change the row.", + "//", + "// Requires additionalProperties: true on the entity in config/registry/schemas/.", + "// The registry re-validates the MERGED document on update, and that document", + "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", + "// extraneous, with the field names it injected itself.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"update accepted\", () => {", + " if (msg.includes(\"extraneous key\")) {", + " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", + " + \"additionalProperties on this entity and restart the registry: \" + msg);", + " }", + " pm.expect(pm.response.code, msg).to.eql(200);", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + "});" + ] + } + } + ] + }, + { + "name": "10. Registry \u2014 update the mandi upstream URL (PUT)", + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant/{{targetOsid}}", + "body": { + "mode": "raw", + "raw": "{\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The registry addresses a record by the osid it assigned on write, not by", + "// participantId. So an update has to look the osid up first. Doing it here rather", + "// than in a preceding request keeps this one runnable on its own.", + "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", + "const want = pm.variables.replaceIn(\"{{mandiProviderId}}\");", + "const filters = {}; filters[\"participantId\"] = { eq: want };", + "", + "pm.sendRequest({", + " url: base + \"/Participant/search\",", + " method: \"POST\",", + " header: { \"Content-Type\": \"application/json\" },", + " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", + "}, function (err, res) {", + " if (err) { console.log(\"osid lookup failed: \" + err); return; }", + " const rows = ((res.json() || {}).data) || [];", + " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", + " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A partial body MERGES: fields you leave out keep their stored values. That is", + "// what makes this safe to re-run -- it writes the same value the variable already", + "// holds, so a default run changes nothing. Change the variable to change the row.", + "//", + "// Requires additionalProperties: true on the entity in config/registry/schemas/.", + "// The registry re-validates the MERGED document on update, and that document", + "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", + "// extraneous, with the field names it injected itself.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"update accepted\", () => {", + " if (msg.includes(\"extraneous key\")) {", + " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", + " + \"additionalProperties on this entity and restart the registry: \" + msg);", + " }", + " pm.expect(pm.response.code, msg).to.eql(200);", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + "});" + ] + } + } + ] + }, + { + "name": "11. Registry \u2014 update the weather binding's call plan (PUT)", + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema/{{targetOsid}}", + "body": { + "mode": "raw", + "raw": "{\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Change a call plan without recreating the binding: a new mapping URL, a different path, a longer timeout, more retries.\n\nUnlike the upstream update this DOES send the whole actions list, because actions is a list and replacing one entry means sending the list. So it also needs additionalProperties: true on ActionBinding, not just on ProviderSchema.\n\nRe-runnable for the same reason: it writes back what the variables already hold." + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The registry addresses a record by the osid it assigned on write, not by", + "// bindingKey. So an update has to look the osid up first. Doing it here rather", + "// than in a preceding request keeps this one runnable on its own.", + "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", + "const want = pm.variables.replaceIn(\"{{weatherBindingKey}}\");", + "const filters = {}; filters[\"bindingKey\"] = { eq: want };", + "", + "pm.sendRequest({", + " url: base + \"/ProviderSchema/search\",", + " method: \"POST\",", + " header: { \"Content-Type\": \"application/json\" },", + " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", + "}, function (err, res) {", + " if (err) { console.log(\"osid lookup failed: \" + err); return; }", + " const rows = ((res.json() || {}).data) || [];", + " if (!rows.length) { console.log(\"no ProviderSchema matching \" + want); return; }", + " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A partial body MERGES: fields you leave out keep their stored values. That is", + "// what makes this safe to re-run -- it writes the same value the variable already", + "// holds, so a default run changes nothing. Change the variable to change the row.", + "//", + "// Requires additionalProperties: true on the entity in config/registry/schemas/.", + "// The registry re-validates the MERGED document on update, and that document", + "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", + "// extraneous, with the field names it injected itself.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"update accepted\", () => {", + " if (msg.includes(\"extraneous key\")) {", + " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", + " + \"additionalProperties on this entity and restart the registry: \" + msg);", + " }", + " pm.expect(pm.response.code, msg).to.eql(200);", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + "});" + ] + } + } + ] + }, + { + "name": "12. Registry \u2014 search participants", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{registryUrl}}/Participant/search", + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Search takes NO token -- it is the one registry call a network peer actually needs. That is also why the dev deployment keeps the whole service off the public edge: SunbirdRC uses POST for both reads and writes, so no method rule separates this from a create.\n\nAn empty filters object returns everything. Narrow it with e.g. {\"filters\":{\"participantId\":{\"eq\":\"exp.oan.dev\"}}}." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().data;", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "", + "// The three adapter identities must exist WITH keys, or nothing can sign.", + "const nodes = rows.filter(r => r.type === \"node\");", + "pm.test(\"three adapter identities, each with a signing key\", () => {", + " pm.expect(nodes.length).to.be.at.least(3);", + " nodes.forEach(n => pm.expect((n.keys || []).length, n.participantId).to.be.above(0));", + "});", + "", + "pm.test(\"both upstreams are registered as type upstream\", () => {", + " [pm.variables.get(\"providerId\"), pm.variables.get(\"mandiProviderId\")].forEach(id => {", + " const up = rows.filter(r => r.participantId === id);", + " pm.expect(up.length, \"no row for \" + id).to.eql(1);", + " pm.expect(up[0].type, id).to.eql(\"upstream\");", + " });", + "});" + ] + } + } + ] + }, + { + "name": "13. Registry \u2014 search provider bindings", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{registryUrl}}/ProviderSchema/search", + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "The call plans, one row per capability. The two differ in path and mapping URL and point at different upstreams -- which is what lets one provider adapter serve both without knowing anything about either." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().data;", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const keys = rows.map(r => r.bindingKey);", + "", + "pm.test(\"both capability bindings exist\", () => {", + " pm.expect(keys).to.include(pm.variables.get(\"weatherBindingKey\"));", + " pm.expect(keys).to.include(pm.variables.get(\"mandiBindingKey\"));", + "});", + "", + "// The call plans differ, which is the point.", + "pm.test(\"each binding carries its own path\", () => {", + " const paths = {};", + " rows.forEach(r => { paths[r.bindingKey] = (r.actions || [])[0] && r.actions[0].path; });", + " pm.expect(paths[pm.variables.get(\"weatherBindingKey\")]).to.eql(pm.variables.get(\"weatherPath\"));", + " pm.expect(paths[pm.variables.get(\"mandiBindingKey\")]).to.eql(pm.variables.get(\"mandiPath\"));", + "});" + ] + } + } + ] + }, + { + "name": "14. Publish \u2014 weather catalogue", "request": { "method": "POST", "header": [ @@ -75,7 +996,7 @@ ] }, { - "name": "2. Publish \u2014 mandi catalogue", + "name": "15. Publish \u2014 mandi catalogue", "request": { "method": "POST", "header": [ @@ -115,7 +1036,7 @@ ] }, { - "name": "3. Discover \u2014 weather", + "name": "16. Discover \u2014 weather", "request": { "method": "POST", "header": [ @@ -157,7 +1078,7 @@ ] }, { - "name": "4. Discover \u2014 mandi", + "name": "17. Discover \u2014 mandi", "request": { "method": "POST", "header": [ @@ -208,7 +1129,7 @@ ] }, { - "name": "5. Select \u2014 weather, per-day forecast", + "name": "18. Select \u2014 weather, per-day forecast", "request": { "method": "POST", "header": [ @@ -273,7 +1194,7 @@ ] }, { - "name": "6. Select \u2014 mandi, prices per market day", + "name": "19. Select \u2014 mandi, prices per market day", "request": { "method": "POST", "header": [ diff --git a/docker-deployment/postman-collection/OAN-dev.postman_environment.json b/docker-deployment/postman-collection/OAN-dev.postman_environment.json new file mode 100644 index 0000000..db80d85 --- /dev/null +++ b/docker-deployment/postman-collection/OAN-dev.postman_environment.json @@ -0,0 +1,200 @@ +{ + "id": "oan-dev-environment", + "name": "OAN dev", + "values": [ + { + "key": "expAdapterUrl", + "value": "http://localhost:9202", + "enabled": true, + "type": "default", + "description": "Takes unsigned requests -- the app is inside the trust boundary." + }, + { + "key": "networkAdapterUrl", + "value": "http://localhost:9201", + "enabled": true, + "type": "default", + "description": "The network layer adapter -- the peer-facing surface. No request in this collection uses it: discover reaches it via the experience adapter, and publish via the provider adapter. It is here because it is the one adapter besides exp that a deployment exposes publicly, and because its /publish and /discover both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not scripted here." + }, + { + "key": "providerAdapterUrl", + "value": "http://localhost:9200", + "enabled": true, + "type": "default", + "description": "Where publish enters." + }, + { + "key": "registryUrl", + "value": "http://localhost:8081/api/v1", + "enabled": true, + "type": "default", + "description": "Loopback. Tunnel to the VM if it is not this machine." + }, + { + "key": "keycloakUrl", + "value": "http://localhost:8080", + "enabled": true, + "type": "default", + "description": "Issues the write token." + }, + { + "key": "discoveryUrl", + "value": "http://localhost:8090", + "enabled": true, + "type": "default" + }, + { + "key": "keycloakRealm", + "value": "sunbird-rc", + "enabled": true, + "type": "default" + }, + { + "key": "keycloakClientId", + "value": "registry-frontend", + "enabled": true, + "type": "default" + }, + { + "key": "registryUser", + "value": "no-user", + "enabled": true, + "type": "default" + }, + { + "key": "registryPassword", + "value": "no-user-password", + "enabled": true, + "type": "default" + }, + { + "key": "expNodeId", + "value": "exp.oan.dev", + "enabled": true, + "type": "default" + }, + { + "key": "expNodeKey", + "value": "", + "enabled": true, + "type": "default", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "networkNodeId", + "value": "network.oan.dev", + "enabled": true, + "type": "default" + }, + { + "key": "networkNodeKey", + "value": "", + "enabled": true, + "type": "default", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "providerNodeId", + "value": "provider.oan.dev", + "enabled": true, + "type": "default" + }, + { + "key": "providerNodeKey", + "value": "", + "enabled": true, + "type": "default", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "providerId", + "value": "mausamgram-mock", + "enabled": true, + "type": "default", + "description": "The weather upstream. Half of its binding key." + }, + { + "key": "weatherCapability", + "value": "openagrinet:WeatherObservation", + "enabled": true, + "type": "default" + }, + { + "key": "weatherBindingKey", + "value": "mausamgram-mock|openagrinet:WeatherObservation", + "enabled": true, + "type": "default", + "description": "Used to look up the binding's osid for the update." + }, + { + "key": "weatherBaseUrl", + "value": "http://mockimd:9100", + "enabled": true, + "type": "default", + "description": "A compose service name: reached from inside the network." + }, + { + "key": "weatherPath", + "value": "/get-daily", + "enabled": true, + "type": "default" + }, + { + "key": "weatherMappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml", + "enabled": true, + "type": "default" + }, + { + "key": "mandiProviderId", + "value": "agmarknet-mock", + "enabled": true, + "type": "default", + "description": "The mandi upstream. Half of its binding key." + }, + { + "key": "mandiCapability", + "value": "openagrinet:MandiPrice", + "enabled": true, + "type": "default" + }, + { + "key": "mandiBindingKey", + "value": "agmarknet-mock|openagrinet:MandiPrice", + "enabled": true, + "type": "default" + }, + { + "key": "mandiBaseUrl", + "value": "http://mockagmarknet:9101", + "enabled": true, + "type": "default" + }, + { + "key": "mandiPath", + "value": "/v1/fetch-agmarknet-vistaar", + "enabled": true, + "type": "default" + }, + { + "key": "mandiMappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml", + "enabled": true, + "type": "default" + }, + { + "key": "networkId", + "value": "oan-dev", + "enabled": true, + "type": "default" + }, + { + "key": "domain", + "value": "oan-dev", + "enabled": true, + "type": "default" + } + ], + "_postman_variable_scope": "environment", + "_postman_exported_using": "hand-written, tracked in this repo" +} diff --git a/docker-deployment/postman-collection/README.md b/docker-deployment/postman-collection/README.md index 27848f9..5b15651 100644 --- a/docker-deployment/postman-collection/README.md +++ b/docker-deployment/postman-collection/README.md @@ -1,65 +1,87 @@ # Postman collection -`OAN-dev-flow.postman_collection.json` — publish, discover and select, for -each of the two capabilities, against a stack brought up from the compose file -beside it. Six requests, 32 assertions. +Two files. Import both. -Import it and run it. **There is nothing to fill in.** Every variable is -prefilled with what this deployment actually uses, including the two provider -ids, so the binding keys in the payloads already match the adapter that will -serve them. + OAN-dev-flow.postman_collection.json the requests + OAN-dev.postman_environment.json where your deployment's URLs go -## There are no registry requests here +**The collection alone works against a tunnel.** Every URL variable defaults to +loopback, because the stack publishes its ports on the VM's loopback only: -Deliberately. The registry has no route through the gateway and publishes on -loopback only, so a Postman request could not create or read a row from -outside the VM. `bin/setup.py` seeds all of it — five participants and both -capability bindings — which is what makes this collection short. + ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ + -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm -So the prerequisite is the stack being up the normal way: +**The environment is how you point it somewhere else.** Import it, select it in +the environment dropdown, and edit the six URLs at the top — the experience, +network and provider adapters, the registry, Keycloak and discovery. Postman +resolves an environment variable ahead of a collection variable of the same +name, so nothing in the collection needs touching and the loopback defaults +stay intact for the next person. - make up +No VM hostname or address is committed in either file. Deployment addresses are +shared separately, and the environment file is the place to put them. -## Run them in order the first time +## The 19 requests -Requests 1 and 2 publish the catalogues that 3 and 4 search for. After that -any request works on its own. Re-running is safe: publish is idempotent from -the caller's point of view, and nothing here writes to the registry. + 1 get a write token saves {{token}} + 2-4 create the exp / network / provider nodes + 5-8 create both upstreams and both bindings + 9-10 update an upstream's URL (PUT) + 11 update a binding's call plan (PUT) + 12-13 search participants / provider bindings + 14-19 publish, discover, select -- both capabilities -## What the assertions actually check +Run in order the first time: request 1 issues the token that 2-11 need, and the +publish requests seed the catalogues discover looks for. -Enough that a green run means the stack is healthy, not just answering: +## A default run changes nothing -- publish comes back `ACCEPTED` -- discover returns at least one catalogue, and for mandi that the specific - catalogue just published is the one found -- a discovered catalogue advertises `OnDemand` and carries no prices — the - pack forbids them in that mode -- select answers with one resource per forecast day, and one per price record -- the mandi answer is in `Direct` mode with every field the pack requires of - it, its prices are numbers rather than the strings the upstream sends, and - `arrivalDate` is ISO rather than the `dd-MM-yyyy` that arrived -- a record with no minimum or maximum omits those fields instead of sending - nulls — the last mock record is built that way on purpose -- status codes come from the spec's `DRAFT|ACTIVE|CLOSED` enum -- every resource carries a `quantity` -- the weather answer names no party in either direction, and its offer - references only resources actually returned +That is deliberate, so the collection is safe to re-run against a live stack. -## Requests 5 and 6 are the interesting pair +- **The creates** report "already present". The registry is append-only -- no + update on create, and a soft delete keeps the unique index -- so a second run + cannot succeed. The test accepts a duplicate and fails anything else, so a + validation error or a bad token is still caught. +- **The updates** write back the same value the variable already holds. Change + a variable to actually change a row. -They hit **the same endpoint on the same adapter**, and different domain -packages answer them. Each provider step builds a binding key from the payload -it is given — provider id plus capability `@type` — serves it if the key is -its own, and passes it through untouched if not. Nothing routes by URL, by -path or by domain, which is what lets one adapter host both capabilities and -what makes adding a third a config change. +## What the updates need -## Tunnelling +`PUT /api/v1/{Entity}/{osid}` works, and a **partial body merges** -- send only +`baseUrl` and the name, type and status keep their stored values. -The ports are on the VM's loopback. From a workstation: +Two things follow from how the registry implements it: - ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 -N you@the-vm +- **The URL takes an osid, not a participantId.** The registry addresses a + record by the id it assigned on write. Each PUT resolves that itself in a + pre-request script, so the request still runs on its own. +- **The entity schema has to permit additional properties.** The registry + re-validates the *merged* document, and that document carries the `osid`, + `osUpdatedAt` and `osOwner` it injected itself -- which `additionalProperties: + false` rejects as extraneous, naming its own fields. `Participant`, + `ProviderSchema` and `ActionBinding` in `config/registry/schemas/` allow them + for this reason. `PublicKey` deliberately does not: nothing here updates key + material, and a partial PUT that omits `keys` never re-validates it. -Those two are all the collection needs: 9202 is the experience adapter, 9200 -the provider adapter, which is where a publish enters. +Schemas are read at startup, so a change there needs the registry restarted. + +## Filling in the node keys + +`expNodeKey`, `networkNodeKey` and `providerNodeKey` ship blank, because the +keypairs are generated per deployment into `keys/keys.json` on the host. +`bin/setup.py` already created those three rows, so requests 2-4 are normally +not needed at all -- they are here to show what a node record looks like. With +the keys blank they report "awaiting a key" rather than failing the run. + +If you do fill them in, use the public half exactly as `keys/keys.json` holds +it: bare base64, no encoding label. A node created with a key the adapter does +not hold produces signatures nobody can verify, and the id cannot be reclaimed. + +## networkAdapterUrl + +Present as a variable, used by no request. Discover reaches the network adapter +through the experience adapter and publish through the provider adapter, so +nothing here calls it directly. It is there because it is the other adapter a +deployment exposes publicly: its `/publish` and `/discover` both verify +signatures, so a network peer calls it directly. Signing is not something +Postman does, so those calls are not scripted. From 615817b8a97b79ee27cf8bfc4036aded859c7e0c Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 12:52:47 +0530 Subject: [PATCH 40/81] fix: let the registry schemas accept an update [OpenAgriNet/network-adapter#4] I had been asserting this registry cannot update a record. It can -- PUT /api/v1/{Entity}/{osid} works, and a partial body merges, so changing a mock's baseUrl is a one-field request. What blocked it was this repo's own schemas. The registry re-validates the MERGED document on an update, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself. With additionalProperties: false it rejects its own fields as extraneous, naming them in the error -- which reads like a bad request body when the body was fine. So three definitions now permit additional properties, each with the reason in its description so it does not get "fixed" back: Participant repoint an upstream, or correct a node's baseUrl ProviderSchema change a binding's call plan ActionBinding because replacing one action means sending the actions list PublicKey stays strict deliberately. Nothing updates key material through these requests, and a partial PUT that omits `keys` never re-validates that sub-object -- verified: a node's baseUrl updates cleanly with PublicKey still closed. THE COST, since it is real: create-time validation for these three entities no longer catches a misspelt field name. SunbirdRC validates create and update against the same schema, so strict-on-create and permissive-on-update is not expressible. A typo that used to be a 400 now writes a record with a stray property, and the registry cannot delete it. Tested in oan-local: upstream baseUrl, node baseUrl and a binding's timeout and retry count all updated and read back correctly, and the full collection still passes. Schemas are read at startup, so this needs the registry restarted to take effect. --- .../config/registry/schemas/Participant.json | 4 ++-- .../config/registry/schemas/ProviderSchema.json | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-deployment/config/registry/schemas/Participant.json b/docker-deployment/config/registry/schemas/Participant.json index 39c93e1..c8cff80 100644 --- a/docker-deployment/config/registry/schemas/Participant.json +++ b/docker-deployment/config/registry/schemas/Participant.json @@ -12,9 +12,9 @@ }, "definitions": { "Participant": { - "description": "Someone the network deals with. `type` says which kind, and decides which of the remaining fields apply: a node speaks Beckn and is addressed by its participantId; an upstream is an ordinary API our adapter calls.", + "description": "Someone the network deals with. `type` says which kind, and decides which of the remaining fields apply: a node speaks Beckn and is addressed by its participantId; an upstream is an ordinary API our adapter calls. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "required": [ "participantId", "name", diff --git a/docker-deployment/config/registry/schemas/ProviderSchema.json b/docker-deployment/config/registry/schemas/ProviderSchema.json index 16f5200..7d25d31 100644 --- a/docker-deployment/config/registry/schemas/ProviderSchema.json +++ b/docker-deployment/config/registry/schemas/ProviderSchema.json @@ -8,9 +8,9 @@ "definitions": { "ProviderSchema": { - "description": "One row is one provider and one capability. What varies per Beckn action — the URL, the method, the mapping, the timeout — varies inside actions[].", + "description": "One row is one provider and one capability. What varies per Beckn action — the URL, the method, the mapping, the timeout — varies inside actions[]. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "required": ["bindingKey", "participantId", "capabilityCode", "status", "actions"], "properties": { "bindingKey": { @@ -33,9 +33,9 @@ }, "ActionBinding": { - "description": "How to call this provider for one Beckn action: where, how, and with which mapping. Anything the upstream needs that the Beckn body cannot express is the adapter plugin's work, not a field here. status is per action, so one can be retired without touching the others.", + "description": "How to call this provider for one Beckn action: where, how, and with which mapping. Anything the upstream needs that the Beckn body cannot express is the adapter plugin's work, not a field here. status is per action, so one can be retired without touching the others. Permits additional properties because the registry re-validates the MERGED document on a PUT, and that document carries the osid, osUpdatedAt and osOwner the registry injected itself -- which a strict schema rejects as extraneous. Do not set this back to false without removing the update requests from the Postman collection.", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "required": ["action", "method", "path", "mappings", "status"], "properties": { "action": { "$ref": "#/definitions/Action" }, From 1eb98da7b083b89ea35962fbf915eb6a54588a5e Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:12:37 +0530 Subject: [PATCH 41/81] feat: point the discovery chart at the published image [#53] CI publishes ghcr.io/openagrinet/discovery-service now. It used to build and scan an image and push nothing, which is the whole reason repository was empty and the render was made to fail on it. tag stays empty rather than pinned. Empty falls through to Chart.AppVersion, so the chart ships pointing at the app version it was written against, and an environment wanting a different build says so in its own values file. Pinning it here would only be right once the chart and the app stop versioning together. Records that the package is private, which the old comment had no reason to. A cluster needs a docker-registry secret named in pullSecrets, and without it the deploy looks clean while the pod sits in ImagePullBackOff -- the same late-surfacing failure the empty-repository guard exists to prevent. --- charts/discovery/examples/discovery.dev.yaml | 8 ++++--- charts/discovery/values.yaml | 23 +++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/charts/discovery/examples/discovery.dev.yaml b/charts/discovery/examples/discovery.dev.yaml index 48a0f8e..6fa09cd 100644 --- a/charts/discovery/examples/discovery.dev.yaml +++ b/charts/discovery/examples/discovery.dev.yaml @@ -12,13 +12,15 @@ # No fullnameOverride needed: the chart is named "discovery", so a release # named "discovery" already produces Service/discovery. -# TODO: no image is published for discovery-service yet - CI builds and scans -# one but pushes nothing. Fill this in with whatever the first published tag is; -# the render fails until then, on purpose. +# The package is private, so a cluster needs a docker-registry secret named in +# pullSecrets as well as this. Pinned rather than left to Chart.AppVersion, +# because a dev environment is where you want to run a build the chart was not +# written against. image: registry: ghcr.io repository: openagrinet/discovery-service tag: "0.1.0" + pullSecrets: [] app: # The network this deployment serves. It fills an empty diff --git a/charts/discovery/values.yaml b/charts/discovery/values.yaml index 90c698f..90aa4ff 100644 --- a/charts/discovery/values.yaml +++ b/charts/discovery/values.yaml @@ -27,11 +27,22 @@ replicaCount: 1 # --------------------------------------------------------------------------- # Image # -# repository is EMPTY on purpose. discovery-service's CI builds an image and -# scans it but pushes it nowhere, so there is no tag to default to; the compose -# stack builds from the working tree. The render fails while this is empty -# rather than producing "ghcr.io/:0.1.0", which Helm and the API server both -# accept and which only surfaces later as an ImagePullBackOff. +# ghcr.io/openagrinet/discovery-service, which CI now publishes -- it used to +# build and scan an image and push nothing, which is why this was empty. +# +# tag stays empty, and that is not an oversight: empty falls through to +# Chart.AppVersion, so the chart ships pointing at the app version it was +# written against, and an environment that wants a different build says so in +# its own values file. Pin it here only when the chart and the app stop +# versioning together. +# +# The package is PRIVATE. A cluster pulling it needs a docker-registry secret +# and that secret named in pullSecrets below -- otherwise the deploy looks +# clean and the pod sits in ImagePullBackOff: +# +# kubectl create secret docker-registry ghcr \ +# --docker-server=ghcr.io --docker-username= \ +# --docker-password= # # The runtime is gcr.io/distroless/static-debian12:nonroot - no shell, no # package manager, uid 65532. That is what makes the security context defaults @@ -39,7 +50,7 @@ replicaCount: 1 # --------------------------------------------------------------------------- image: registry: ghcr.io - repository: "" + repository: openagrinet/discovery-service tag: "" digest: "" pullPolicy: IfNotPresent From 7ed5baf3493d9ce5d2139afa670c7464bb7c102b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 13:27:59 +0530 Subject: [PATCH 42/81] feat: let an upstream be created with its own signing key [OpenAgriNet/network-adapter#4] The two upstream creates now carry a keys block, blank by default and dropped when the variable is empty, so an upstream can be registered with a signing key of its own. WHY THIS IS ALLOWED, since it reads like a contradiction: an upstream is documented as the thing that signs nothing. The Participant schema declares keys for every type, and its only conditional is `if type == "node" then require role and keys` with no else -- so that branch adds requirements for a node and never forbids keys on an upstream. The adapter accepts such a key as a signer too, for two reasons that both had to hold: the signature lookup filters on participantId alone and never compares type, and isSigning() treats an absent `use` as signing, which matters because this schema drops `use` and lets `alg` carry the purpose. WHY IT IS WORTH HAVING. With a key on that row the provider signs its own catalogue and posts /publish straight at the NETWORK adapter, which verifies against the row. The provider adapter leaves the publish path, and with it the unauthenticated /publish it otherwise has to expose -- the endpoint the gateway carries a deny rule for. Verified end to end rather than reasoned about: an upstream record created with an ed25519 key signed a catalogue and the network adapter answered catalog/on_publish ACCEPTED, while a wrong key, a body tampered with after signing, and a missing Authorization header each came back 401. So the verification really runs and the 200 meant something. The block is stripped rather than sent empty because the mocks have no keypair, and an empty string fails the schema's ^[A-Za-z0-9+/]{43}=$ pattern -- which would refuse the create for a reason unrelated to what the caller was doing. Both paths are tested: blank leaves the create validating (it reaches the duplicate-key check), and a supplied key is present in the sent body. Adds weatherProviderKey and mandiProviderKey to the collection and to the environment file, and documents the Authorization header a provider has to produce -- keyId of participantId|osid|ed25519, over a BLAKE2b-512 digest, which is what "BLAKE-512" in the signing string means. --- .../OAN-dev-flow.postman_collection.json | 84 ++++++++++++++++++- .../OAN-dev.postman_environment.json | 14 ++++ .../postman-collection/README.md | 47 +++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index 90727c4..d20367c 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -101,6 +101,11 @@ "value": "mausamgram-mock|openagrinet:WeatherObservation", "description": "Used to look up the binding's osid for the update." }, + { + "key": "weatherProviderKey", + "value": "", + "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the weather provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." + }, { "key": "weatherBaseUrl", "value": "http://mockimd:9100", @@ -127,6 +132,11 @@ "key": "mandiBindingKey", "value": "agmarknet-mock|openagrinet:MandiPrice" }, + { + "key": "mandiProviderKey", + "value": "", + "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the mandi provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." + }, { "key": "mandiBaseUrl", "value": "http://mockagmarknet:9101" @@ -396,7 +406,7 @@ "url": "{{registryUrl}}/Participant", "body": { "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", + "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{weatherProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -406,6 +416,41 @@ "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." }, "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The keys block below is OPTIONAL on an upstream, and blank by default.", + "//", + "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", + "// only conditional is `if type == \"node\" then require role and keys`, and it has", + "// no else -- so it adds requirements for a node and never forbids them on an", + "// upstream. The adapter accepts such a key as a signer too: the signature lookup", + "// filters on participantId alone and never compares type, and isSigning() treats", + "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", + "//", + "// WHY YOU WOULD WANT IT. With a key on this record the weather provider can sign its", + "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", + "// against this row. That takes the provider adapter out of the publish path", + "// entirely -- and with it the unauthenticated /publish it has to expose.", + "//", + "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", + "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", + "// refused for a reason that has nothing to do with what you were trying to do.", + "//", + "// Set weatherProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", + "// you create the record: the registry is append-only, so a partial PUT can add a", + "// keys array later but cannot remove or replace one.", + "const key = (pm.variables.get(\"weatherProviderKey\") || \"\").trim();", + "if (!key) {", + " const body = JSON.parse(pm.request.body.raw);", + " delete body.keys;", + " pm.request.body.update(JSON.stringify(body, null, 2));", + "}" + ] + } + }, { "listen": "test", "script": { @@ -510,7 +555,7 @@ "url": "{{registryUrl}}/Participant", "body": { "mode": "raw", - "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", + "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{mandiProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" @@ -520,6 +565,41 @@ "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." }, "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The keys block below is OPTIONAL on an upstream, and blank by default.", + "//", + "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", + "// only conditional is `if type == \"node\" then require role and keys`, and it has", + "// no else -- so it adds requirements for a node and never forbids them on an", + "// upstream. The adapter accepts such a key as a signer too: the signature lookup", + "// filters on participantId alone and never compares type, and isSigning() treats", + "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", + "//", + "// WHY YOU WOULD WANT IT. With a key on this record the mandi provider can sign its", + "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", + "// against this row. That takes the provider adapter out of the publish path", + "// entirely -- and with it the unauthenticated /publish it has to expose.", + "//", + "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", + "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", + "// refused for a reason that has nothing to do with what you were trying to do.", + "//", + "// Set mandiProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", + "// you create the record: the registry is append-only, so a partial PUT can add a", + "// keys array later but cannot remove or replace one.", + "const key = (pm.variables.get(\"mandiProviderKey\") || \"\").trim();", + "if (!key) {", + " const body = JSON.parse(pm.request.body.raw);", + " delete body.keys;", + " pm.request.body.update(JSON.stringify(body, null, 2));", + "}" + ] + } + }, { "listen": "test", "script": { diff --git a/docker-deployment/postman-collection/OAN-dev.postman_environment.json b/docker-deployment/postman-collection/OAN-dev.postman_environment.json index db80d85..903884e 100644 --- a/docker-deployment/postman-collection/OAN-dev.postman_environment.json +++ b/docker-deployment/postman-collection/OAN-dev.postman_environment.json @@ -126,6 +126,13 @@ "type": "default", "description": "Used to look up the binding's osid for the update." }, + { + "key": "weatherProviderKey", + "value": "", + "enabled": true, + "type": "default", + "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the weather provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." + }, { "key": "weatherBaseUrl", "value": "http://mockimd:9100", @@ -164,6 +171,13 @@ "enabled": true, "type": "default" }, + { + "key": "mandiProviderKey", + "value": "", + "enabled": true, + "type": "default", + "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the mandi provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." + }, { "key": "mandiBaseUrl", "value": "http://mockagmarknet:9101", diff --git a/docker-deployment/postman-collection/README.md b/docker-deployment/postman-collection/README.md index 5b15651..de26535 100644 --- a/docker-deployment/postman-collection/README.md +++ b/docker-deployment/postman-collection/README.md @@ -77,6 +77,53 @@ If you do fill them in, use the public half exactly as `keys/keys.json` holds it: bare base64, no encoding label. A node created with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed. +## Giving an upstream its own signing key + +The two upstream creates carry a `keys` block, blank by default, and drop it +when the variable is empty. Set `weatherProviderKey` or `mandiProviderKey` to a +provider's **public** signing key and the block is sent. + +**Why an upstream may have keys.** The `Participant` schema declares `keys` for +every type. Its only conditional is `if type == "node" then require role and +keys`, and it has no `else` -- so that branch adds requirements for a node and +never forbids keys on an upstream. The adapter accepts such a key as a signer +too: the signature lookup filters on `participantId` alone and never compares +`type`, and `isSigning()` treats an absent `use` as signing, which matters +because this schema drops `use` and lets `alg` carry the purpose. + +**Why you would want it.** With a key on that row the provider can sign its own +catalogue and `POST /publish` straight at the **network** adapter, which +verifies the signature against the row. The provider adapter drops out of the +publish path -- and with it the unauthenticated `/publish` it otherwise has to +expose, which is the whole reason the gateway carries a deny rule for that path. + +Verified end to end: an upstream record created with an `ed25519` key signed a +catalogue and the network adapter answered `catalog/on_publish` `ACCEPTED`, +while a wrong key, a body tampered with after signing, and a missing +`Authorization` header each came back `401`. + +The header a provider has to produce: + + Signature keyId="||ed25519", + algorithm="ed25519",created="",expires="", + headers="(created) (expires) digest",signature="" + +signed over exactly this string -- real newlines, and `BLAKE-512` meaning +BLAKE2b-512, not SHA: + + (created): + (expires): + digest: BLAKE-512= + +The `osid` is the one the registry assigns the key on write, so a provider has +to read it back from a `Participant/search` after registering. + +**Add the keys when you create the record.** A partial PUT can add a `keys` +array later but cannot remove or replace one -- the registry is append-only. And +the value is bare base64 matching `^[A-Za-z0-9+/]{43}=$`, no encoding label: a +`base64:` prefix left on the front fails verification later with a decode error +that points nowhere near the registry. + ## networkAdapterUrl Present as a variable, used by no request. Discover reaches the network adapter From 5dfbc2b4525ede7e5d4e8060cbcdbe716516125e Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Fri, 4 Sep 2026 14:26:05 +0530 Subject: [PATCH 43/81] refactor: group the collection into Registry, Publish, Discover and Select [OpenAgriNet/network-adapter#4] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nineteen requests in one flat list read as a queue to work through rather than four things the stack does. They are now four folders, one per leg of the flow, and each carries a description of what that leg does and what its failures mean -- an empty discover pointing at networkId, a 404 on select pointing at binding keys. FOLDER ORDER IS LOAD-BEARING. Newman walks the item array depth-first in order, so Registry has to come before Publish for the token, and Publish before Discover for the catalogues it searches for. The numeric prefixes make that visible in the UI rather than implied. Requests are renumbered within each folder and lose the prefix the folder now carries: "3. Publish — weather catalogue" becomes "2. Publish/1. Weather catalogue". The gain beyond tidiness is that a folder now runs on its own. Verified each: Select 17 assertions, Discover 8, Publish 7, all passing standalone, which is what you want when you are debugging one leg rather than proving the stack. Full run unchanged at 22 requests and 50 assertions, in folder order, so nothing about the pre-request scripts or the token variable depends on the flat layout. --- .../OAN-dev-flow.postman_collection.json | 2268 +++++++++-------- .../postman-collection/README.md | 27 +- 2 files changed, 1161 insertions(+), 1134 deletions(-) diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index d20367c..e9274e2 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -1,7 +1,7 @@ { "info": { "name": "OAN dev \u2014 registry, publish, discover, select", - "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nRUN IN ORDER THE FIRST TIME. Request 1 issues the token that 2-11 need. The publish requests seed the catalogues that discover searches for.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", + "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nFOUR FOLDERS, AND THEIR ORDER MATTERS. Registry, Publish, Discover, Select. The token in Registry is what the writes after it use, and Publish seeds the catalogues Discover searches for -- so a first run should go top to bottom. After that any folder runs on its own.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "variable": [ @@ -160,1199 +160,1223 @@ ], "item": [ { - "name": "1. Registry \u2014 get a write token", - "request": { - "method": "POST", - "header": [ - { - "key": "X-Forwarded-Host", - "value": "keycloak:8080" - }, - { - "key": "X-Forwarded-Proto", - "value": "http" - } - ], - "url": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", - "body": { - "mode": "urlencoded", - "urlencoded": [ - { - "key": "client_id", - "value": "{{keycloakClientId}}" - }, - { - "key": "grant_type", - "value": "password" - }, - { - "key": "username", - "value": "{{registryUser}}" + "name": "1. Registry", + "item": [ + { + "name": "1. Get a write token", + "request": { + "method": "POST", + "header": [ + { + "key": "X-Forwarded-Host", + "value": "keycloak:8080" + }, + { + "key": "X-Forwarded-Proto", + "value": "http" + } + ], + "url": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", + "body": { + "mode": "urlencoded", + "urlencoded": [ + { + "key": "client_id", + "value": "{{keycloakClientId}}" + }, + { + "key": "grant_type", + "value": "password" + }, + { + "key": "username", + "value": "{{registryUser}}" + }, + { + "key": "password", + "value": "{{registryPassword}}" + } + ] }, + "description": "Every registry WRITE needs this. Searches take no token at all.\n\nTHE TWO X-Forwarded-* HEADERS ARE NOT OPTIONAL, and keycloak:8080 is the container-internal address on purpose -- not whatever port Keycloak is published on. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and every write below returns 401 with an empty body.\n\nSaved to {{token}}, so run this first.\n\nA 500 here usually means Keycloak's realm is missing -- it shares a database with the registry, so wiping the registry volume takes the realm with it. Restarting Keycloak re-imports it." + }, + "event": [ { - "key": "password", - "value": "{{registryPassword}}" + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const j = pm.response.json();", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", + "pm.collectionVariables.set(\"token\", j.access_token);" + ] + } } ] }, - "description": "Every registry WRITE needs this. Searches take no token at all.\n\nTHE TWO X-Forwarded-* HEADERS ARE NOT OPTIONAL, and keycloak:8080 is the container-internal address on purpose -- not whatever port Keycloak is published on. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and every write below returns 401 with an empty body.\n\nSaved to {{token}}, so run this first.\n\nA 500 here usually means Keycloak's realm is missing -- it shares a database with the registry, so wiping the registry volume takes the realm with it. Restarting Keycloak re-imports it." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const j = pm.response.json();", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", - "pm.collectionVariables.set(\"token\", j.access_token);" - ] - } - } - ] - }, - { - "name": "2. Registry \u2014 create the exp node", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "2. Create the exp node", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{expNodeId}}\",\n \"name\": \"OAN experience layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{expNodeId}}\",\n \"role\": \"consumer\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{expNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{expNodeId}}\",\n \"name\": \"OAN experience layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{expNodeId}}\",\n \"role\": \"consumer\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{expNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Same as the other creates, with one extra allowance: this collection ships", + "// the node key blank on deployments where it is not knowable, and an empty key", + "// fails schema validation before the duplicate check is ever reached. That is", + "// reported rather than failed.", + "const key = pm.variables.get(\"expNodeKey\") || \"\";", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, already present, or awaiting a key\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " if (msg.includes(\"duplicate key\")) return; // seeded already", + " if (!key && msg.includes(\"does not match pattern\")) {", + " console.log(\"not attempted: expNodeKey is empty -- paste it from keys/keys.json\");", + " return;", + " }", + " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", + "});" + ] + } } - } + ] }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Same as the other creates, with one extra allowance: this collection ships", - "// the node key blank on deployments where it is not knowable, and an empty key", - "// fails schema validation before the duplicate check is ever reached. That is", - "// reported rather than failed.", - "const key = pm.variables.get(\"expNodeKey\") || \"\";", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, already present, or awaiting a key\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " if (msg.includes(\"duplicate key\")) return; // seeded already", - " if (!key && msg.includes(\"does not match pattern\")) {", - " console.log(\"not attempted: expNodeKey is empty -- paste it from keys/keys.json\");", - " return;", - " }", - " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", - "});" - ] - } - } - ] - }, - { - "name": "3. Registry \u2014 create the network node", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "3. Create the network node", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{networkNodeId}}\",\n \"name\": \"OAN network layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{networkNodeId}}\",\n \"role\": \"network\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{networkNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{networkNodeId}}\",\n \"name\": \"OAN network layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{networkNodeId}}\",\n \"role\": \"network\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{networkNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Same as the other creates, with one extra allowance: this collection ships", + "// the node key blank on deployments where it is not knowable, and an empty key", + "// fails schema validation before the duplicate check is ever reached. That is", + "// reported rather than failed.", + "const key = pm.variables.get(\"networkNodeKey\") || \"\";", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, already present, or awaiting a key\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " if (msg.includes(\"duplicate key\")) return; // seeded already", + " if (!key && msg.includes(\"does not match pattern\")) {", + " console.log(\"not attempted: networkNodeKey is empty -- paste it from keys/keys.json\");", + " return;", + " }", + " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", + "});" + ] + } } - } + ] }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Same as the other creates, with one extra allowance: this collection ships", - "// the node key blank on deployments where it is not knowable, and an empty key", - "// fails schema validation before the duplicate check is ever reached. That is", - "// reported rather than failed.", - "const key = pm.variables.get(\"networkNodeKey\") || \"\";", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, already present, or awaiting a key\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " if (msg.includes(\"duplicate key\")) return; // seeded already", - " if (!key && msg.includes(\"does not match pattern\")) {", - " console.log(\"not attempted: networkNodeKey is empty -- paste it from keys/keys.json\");", - " return;", - " }", - " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", - "});" - ] - } - } - ] - }, - { - "name": "4. Registry \u2014 create the provider node", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "4. Create the provider node", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{providerNodeId}}\",\n \"name\": \"OAN provider layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{providerNodeId}}\",\n \"role\": \"provider\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{providerNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerNodeId}}\",\n \"name\": \"OAN provider layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{providerNodeId}}\",\n \"role\": \"provider\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{providerNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Same as the other creates, with one extra allowance: this collection ships", + "// the node key blank on deployments where it is not knowable, and an empty key", + "// fails schema validation before the duplicate check is ever reached. That is", + "// reported rather than failed.", + "const key = pm.variables.get(\"providerNodeKey\") || \"\";", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, already present, or awaiting a key\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " if (msg.includes(\"duplicate key\")) return; // seeded already", + " if (!key && msg.includes(\"does not match pattern\")) {", + " console.log(\"not attempted: providerNodeKey is empty -- paste it from keys/keys.json\");", + " return;", + " }", + " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", + "});" + ] + } } - } + ] }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Same as the other creates, with one extra allowance: this collection ships", - "// the node key blank on deployments where it is not knowable, and an empty key", - "// fails schema validation before the duplicate check is ever reached. That is", - "// reported rather than failed.", - "const key = pm.variables.get(\"providerNodeKey\") || \"\";", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, already present, or awaiting a key\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " if (msg.includes(\"duplicate key\")) return; // seeded already", - " if (!key && msg.includes(\"does not match pattern\")) {", - " console.log(\"not attempted: providerNodeKey is empty -- paste it from keys/keys.json\");", - " return;", - " }", - " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", - "});" - ] - } - } - ] - }, - { - "name": "5. Registry \u2014 create the weather upstream", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "5. Create the weather upstream", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{weatherProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{weatherProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The keys block below is OPTIONAL on an upstream, and blank by default.", + "//", + "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", + "// only conditional is `if type == \"node\" then require role and keys`, and it has", + "// no else -- so it adds requirements for a node and never forbids them on an", + "// upstream. The adapter accepts such a key as a signer too: the signature lookup", + "// filters on participantId alone and never compares type, and isSigning() treats", + "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", + "//", + "// WHY YOU WOULD WANT IT. With a key on this record the weather provider can sign its", + "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", + "// against this row. That takes the provider adapter out of the publish path", + "// entirely -- and with it the unauthenticated /publish it has to expose.", + "//", + "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", + "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", + "// refused for a reason that has nothing to do with what you were trying to do.", + "//", + "// Set weatherProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", + "// you create the record: the registry is append-only, so a partial PUT can add a", + "// keys array later but cannot remove or replace one.", + "const key = (pm.variables.get(\"weatherProviderKey\") || \"\").trim();", + "if (!key) {", + " const body = JSON.parse(pm.request.body.raw);", + " delete body.keys;", + " pm.request.body.update(JSON.stringify(body, null, 2));", + "}" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } } - } - }, - "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The keys block below is OPTIONAL on an upstream, and blank by default.", - "//", - "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", - "// only conditional is `if type == \"node\" then require role and keys`, and it has", - "// no else -- so it adds requirements for a node and never forbids them on an", - "// upstream. The adapter accepts such a key as a signer too: the signature lookup", - "// filters on participantId alone and never compares type, and isSigning() treats", - "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", - "//", - "// WHY YOU WOULD WANT IT. With a key on this record the weather provider can sign its", - "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", - "// against this row. That takes the provider adapter out of the publish path", - "// entirely -- and with it the unauthenticated /publish it has to expose.", - "//", - "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", - "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", - "// refused for a reason that has nothing to do with what you were trying to do.", - "//", - "// Set weatherProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", - "// you create the record: the registry is append-only, so a partial PUT can add a", - "// keys array later but cannot remove or replace one.", - "const key = (pm.variables.get(\"weatherProviderKey\") || \"\").trim();", - "if (!key) {", - " const body = JSON.parse(pm.request.body.raw);", - " delete body.keys;", - " pm.request.body.update(JSON.stringify(body, null, 2));", - "}" - ] - } + ] }, { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ] - }, - { - "name": "6. Registry \u2014 create the weather capability binding", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "6. Create the weather capability binding", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema", + "body": { + "mode": "raw", + "raw": "{\n \"bindingKey\": \"{{providerId}}|{{weatherCapability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{weatherCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/ProviderSchema", - "body": { - "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{providerId}}|{{weatherCapability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{weatherCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } } - } + ] }, - "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ] - }, - { - "name": "7. Registry \u2014 create the mandi upstream", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "7. Create the mandi upstream", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant", + "body": { + "mode": "raw", + "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{mandiProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{mandiProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The keys block below is OPTIONAL on an upstream, and blank by default.", + "//", + "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", + "// only conditional is `if type == \"node\" then require role and keys`, and it has", + "// no else -- so it adds requirements for a node and never forbids them on an", + "// upstream. The adapter accepts such a key as a signer too: the signature lookup", + "// filters on participantId alone and never compares type, and isSigning() treats", + "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", + "//", + "// WHY YOU WOULD WANT IT. With a key on this record the mandi provider can sign its", + "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", + "// against this row. That takes the provider adapter out of the publish path", + "// entirely -- and with it the unauthenticated /publish it has to expose.", + "//", + "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", + "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", + "// refused for a reason that has nothing to do with what you were trying to do.", + "//", + "// Set mandiProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", + "// you create the record: the registry is append-only, so a partial PUT can add a", + "// keys array later but cannot remove or replace one.", + "const key = (pm.variables.get(\"mandiProviderKey\") || \"\").trim();", + "if (!key) {", + " const body = JSON.parse(pm.request.body.raw);", + " delete body.keys;", + " pm.request.body.update(JSON.stringify(body, null, 2));", + "}" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } } - } - }, - "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The keys block below is OPTIONAL on an upstream, and blank by default.", - "//", - "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", - "// only conditional is `if type == \"node\" then require role and keys`, and it has", - "// no else -- so it adds requirements for a node and never forbids them on an", - "// upstream. The adapter accepts such a key as a signer too: the signature lookup", - "// filters on participantId alone and never compares type, and isSigning() treats", - "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", - "//", - "// WHY YOU WOULD WANT IT. With a key on this record the mandi provider can sign its", - "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", - "// against this row. That takes the provider adapter out of the publish path", - "// entirely -- and with it the unauthenticated /publish it has to expose.", - "//", - "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", - "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", - "// refused for a reason that has nothing to do with what you were trying to do.", - "//", - "// Set mandiProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", - "// you create the record: the registry is append-only, so a partial PUT can add a", - "// keys array later but cannot remove or replace one.", - "const key = (pm.variables.get(\"mandiProviderKey\") || \"\").trim();", - "if (!key) {", - " const body = JSON.parse(pm.request.body.raw);", - " delete body.keys;", - " pm.request.body.update(JSON.stringify(body, null, 2));", - "}" - ] - } + ] }, { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ] - }, - { - "name": "8. Registry \u2014 create the mandi capability binding", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "8. Create the mandi capability binding", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema", + "body": { + "mode": "raw", + "raw": "{\n \"bindingKey\": \"{{mandiProviderId}}|{{mandiCapability}}\",\n \"participantId\": \"{{mandiProviderId}}\",\n \"capabilityCode\": \"{{mandiCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{mandiPath}}\",\n \"mappings\": \"{{mandiMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/ProviderSchema", - "body": { - "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{mandiProviderId}}|{{mandiCapability}}\",\n \"participantId\": \"{{mandiProviderId}}\",\n \"capabilityCode\": \"{{mandiCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{mandiPath}}\",\n \"mappings\": \"{{mandiMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// Append-only: no update on create, and a soft delete keeps the unique index,", + "// so a second run of this cannot succeed -- and must not fail the run either.", + "//", + "// The discriminator is the message, not the status code:", + "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", + "// 400 + \"Validation Exception\" the body is wrong.", + "// 401 + empty body no usable token. See request 1.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"created, or already present\", () => {", + " if (pm.response.code >= 200 && pm.response.code < 300) {", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + " return;", + " }", + " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", + " .to.include(\"duplicate key\");", + "});" + ] + } } - } + ] }, - "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ] - }, - { - "name": "9. Registry \u2014 update the weather upstream URL (PUT)", - "request": { - "method": "PUT", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "9. Update the weather upstream URL (PUT)", + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant/{{targetOsid}}", + "body": { + "mode": "raw", + "raw": "{\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant/{{targetOsid}}", - "body": { - "mode": "raw", - "raw": "{\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The registry addresses a record by the osid it assigned on write, not by", + "// participantId. So an update has to look the osid up first. Doing it here rather", + "// than in a preceding request keeps this one runnable on its own.", + "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", + "const want = pm.variables.replaceIn(\"{{providerId}}\");", + "const filters = {}; filters[\"participantId\"] = { eq: want };", + "", + "pm.sendRequest({", + " url: base + \"/Participant/search\",", + " method: \"POST\",", + " header: { \"Content-Type\": \"application/json\" },", + " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", + "}, function (err, res) {", + " if (err) { console.log(\"osid lookup failed: \" + err); return; }", + " const rows = ((res.json() || {}).data) || [];", + " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", + " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A partial body MERGES: fields you leave out keep their stored values. That is", + "// what makes this safe to re-run -- it writes the same value the variable already", + "// holds, so a default run changes nothing. Change the variable to change the row.", + "//", + "// Requires additionalProperties: true on the entity in config/registry/schemas/.", + "// The registry re-validates the MERGED document on update, and that document", + "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", + "// extraneous, with the field names it injected itself.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"update accepted\", () => {", + " if (msg.includes(\"extraneous key\")) {", + " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", + " + \"additionalProperties on this entity and restart the registry: \" + msg);", + " }", + " pm.expect(pm.response.code, msg).to.eql(200);", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + "});" + ] + } } - } - }, - "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The registry addresses a record by the osid it assigned on write, not by", - "// participantId. So an update has to look the osid up first. Doing it here rather", - "// than in a preceding request keeps this one runnable on its own.", - "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", - "const want = pm.variables.replaceIn(\"{{providerId}}\");", - "const filters = {}; filters[\"participantId\"] = { eq: want };", - "", - "pm.sendRequest({", - " url: base + \"/Participant/search\",", - " method: \"POST\",", - " header: { \"Content-Type\": \"application/json\" },", - " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", - "}, function (err, res) {", - " if (err) { console.log(\"osid lookup failed: \" + err); return; }", - " const rows = ((res.json() || {}).data) || [];", - " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", - " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", - "});" - ] - } + ] }, { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// A partial body MERGES: fields you leave out keep their stored values. That is", - "// what makes this safe to re-run -- it writes the same value the variable already", - "// holds, so a default run changes nothing. Change the variable to change the row.", - "//", - "// Requires additionalProperties: true on the entity in config/registry/schemas/.", - "// The registry re-validates the MERGED document on update, and that document", - "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", - "// extraneous, with the field names it injected itself.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"update accepted\", () => {", - " if (msg.includes(\"extraneous key\")) {", - " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", - " + \"additionalProperties on this entity and restart the registry: \" + msg);", - " }", - " pm.expect(pm.response.code, msg).to.eql(200);", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - "});" - ] - } - } - ] - }, - { - "name": "10. Registry \u2014 update the mandi upstream URL (PUT)", - "request": { - "method": "PUT", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "10. Update the mandi upstream URL (PUT)", + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/Participant/{{targetOsid}}", + "body": { + "mode": "raw", + "raw": "{\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant/{{targetOsid}}", - "body": { - "mode": "raw", - "raw": "{\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The registry addresses a record by the osid it assigned on write, not by", + "// participantId. So an update has to look the osid up first. Doing it here rather", + "// than in a preceding request keeps this one runnable on its own.", + "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", + "const want = pm.variables.replaceIn(\"{{mandiProviderId}}\");", + "const filters = {}; filters[\"participantId\"] = { eq: want };", + "", + "pm.sendRequest({", + " url: base + \"/Participant/search\",", + " method: \"POST\",", + " header: { \"Content-Type\": \"application/json\" },", + " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", + "}, function (err, res) {", + " if (err) { console.log(\"osid lookup failed: \" + err); return; }", + " const rows = ((res.json() || {}).data) || [];", + " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", + " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A partial body MERGES: fields you leave out keep their stored values. That is", + "// what makes this safe to re-run -- it writes the same value the variable already", + "// holds, so a default run changes nothing. Change the variable to change the row.", + "//", + "// Requires additionalProperties: true on the entity in config/registry/schemas/.", + "// The registry re-validates the MERGED document on update, and that document", + "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", + "// extraneous, with the field names it injected itself.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"update accepted\", () => {", + " if (msg.includes(\"extraneous key\")) {", + " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", + " + \"additionalProperties on this entity and restart the registry: \" + msg);", + " }", + " pm.expect(pm.response.code, msg).to.eql(200);", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + "});" + ] + } } - } - }, - "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The registry addresses a record by the osid it assigned on write, not by", - "// participantId. So an update has to look the osid up first. Doing it here rather", - "// than in a preceding request keeps this one runnable on its own.", - "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", - "const want = pm.variables.replaceIn(\"{{mandiProviderId}}\");", - "const filters = {}; filters[\"participantId\"] = { eq: want };", - "", - "pm.sendRequest({", - " url: base + \"/Participant/search\",", - " method: \"POST\",", - " header: { \"Content-Type\": \"application/json\" },", - " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", - "}, function (err, res) {", - " if (err) { console.log(\"osid lookup failed: \" + err); return; }", - " const rows = ((res.json() || {}).data) || [];", - " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", - " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", - "});" - ] - } + ] }, { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// A partial body MERGES: fields you leave out keep their stored values. That is", - "// what makes this safe to re-run -- it writes the same value the variable already", - "// holds, so a default run changes nothing. Change the variable to change the row.", - "//", - "// Requires additionalProperties: true on the entity in config/registry/schemas/.", - "// The registry re-validates the MERGED document on update, and that document", - "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", - "// extraneous, with the field names it injected itself.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"update accepted\", () => {", - " if (msg.includes(\"extraneous key\")) {", - " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", - " + \"additionalProperties on this entity and restart the registry: \" + msg);", - " }", - " pm.expect(pm.response.code, msg).to.eql(200);", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - "});" - ] - } - } - ] - }, - { - "name": "11. Registry \u2014 update the weather binding's call plan (PUT)", - "request": { - "method": "PUT", - "header": [ - { - "key": "Content-Type", - "value": "application/json" + "name": "11. Update the weather binding's call plan (PUT)", + "request": { + "method": "PUT", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" + } + ], + "url": "{{registryUrl}}/ProviderSchema/{{targetOsid}}", + "body": { + "mode": "raw", + "raw": "{\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Change a call plan without recreating the binding: a new mapping URL, a different path, a longer timeout, more retries.\n\nUnlike the upstream update this DOES send the whole actions list, because actions is a list and replacing one entry means sending the list. So it also needs additionalProperties: true on ActionBinding, not just on ProviderSchema.\n\nRe-runnable for the same reason: it writes back what the variables already hold." }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/ProviderSchema/{{targetOsid}}", - "body": { - "mode": "raw", - "raw": "{\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// The registry addresses a record by the osid it assigned on write, not by", + "// bindingKey. So an update has to look the osid up first. Doing it here rather", + "// than in a preceding request keeps this one runnable on its own.", + "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", + "const want = pm.variables.replaceIn(\"{{weatherBindingKey}}\");", + "const filters = {}; filters[\"bindingKey\"] = { eq: want };", + "", + "pm.sendRequest({", + " url: base + \"/ProviderSchema/search\",", + " method: \"POST\",", + " header: { \"Content-Type\": \"application/json\" },", + " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", + "}, function (err, res) {", + " if (err) { console.log(\"osid lookup failed: \" + err); return; }", + " const rows = ((res.json() || {}).data) || [];", + " if (!rows.length) { console.log(\"no ProviderSchema matching \" + want); return; }", + " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "// A partial body MERGES: fields you leave out keep their stored values. That is", + "// what makes this safe to re-run -- it writes the same value the variable already", + "// holds, so a default run changes nothing. Change the variable to change the row.", + "//", + "// Requires additionalProperties: true on the entity in config/registry/schemas/.", + "// The registry re-validates the MERGED document on update, and that document", + "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", + "// extraneous, with the field names it injected itself.", + "let body = {};", + "try { body = pm.response.json(); } catch (e) {}", + "const p = body.params || {};", + "const msg = p.errmsg || \"\";", + "", + "pm.test(\"update accepted\", () => {", + " if (msg.includes(\"extraneous key\")) {", + " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", + " + \"additionalProperties on this entity and restart the registry: \" + msg);", + " }", + " pm.expect(pm.response.code, msg).to.eql(200);", + " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", + "});" + ] + } } - } - }, - "description": "Change a call plan without recreating the binding: a new mapping URL, a different path, a longer timeout, more retries.\n\nUnlike the upstream update this DOES send the whole actions list, because actions is a list and replacing one entry means sending the list. So it also needs additionalProperties: true on ActionBinding, not just on ProviderSchema.\n\nRe-runnable for the same reason: it writes back what the variables already hold." - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The registry addresses a record by the osid it assigned on write, not by", - "// bindingKey. So an update has to look the osid up first. Doing it here rather", - "// than in a preceding request keeps this one runnable on its own.", - "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", - "const want = pm.variables.replaceIn(\"{{weatherBindingKey}}\");", - "const filters = {}; filters[\"bindingKey\"] = { eq: want };", - "", - "pm.sendRequest({", - " url: base + \"/ProviderSchema/search\",", - " method: \"POST\",", - " header: { \"Content-Type\": \"application/json\" },", - " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", - "}, function (err, res) {", - " if (err) { console.log(\"osid lookup failed: \" + err); return; }", - " const rows = ((res.json() || {}).data) || [];", - " if (!rows.length) { console.log(\"no ProviderSchema matching \" + want); return; }", - " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", - "});" - ] - } + ] }, { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// A partial body MERGES: fields you leave out keep their stored values. That is", - "// what makes this safe to re-run -- it writes the same value the variable already", - "// holds, so a default run changes nothing. Change the variable to change the row.", - "//", - "// Requires additionalProperties: true on the entity in config/registry/schemas/.", - "// The registry re-validates the MERGED document on update, and that document", - "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", - "// extraneous, with the field names it injected itself.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"update accepted\", () => {", - " if (msg.includes(\"extraneous key\")) {", - " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", - " + \"additionalProperties on this entity and restart the registry: \" + msg);", - " }", - " pm.expect(pm.response.code, msg).to.eql(200);", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - "});" - ] - } - } - ] - }, - { - "name": "12. Registry \u2014 search participants", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": "{{registryUrl}}/Participant/search", - "body": { - "mode": "raw", - "raw": "{\n \"filters\": {}\n}", - "options": { - "raw": { - "language": "json" + "name": "12. Search participants", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{registryUrl}}/Participant/search", + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Search takes NO token -- it is the one registry call a network peer actually needs. That is also why the dev deployment keeps the whole service off the public edge: SunbirdRC uses POST for both reads and writes, so no method rule separates this from a create.\n\nAn empty filters object returns everything. Narrow it with e.g. {\"filters\":{\"participantId\":{\"eq\":\"exp.oan.dev\"}}}." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().data;", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "", + "// The three adapter identities must exist WITH keys, or nothing can sign.", + "const nodes = rows.filter(r => r.type === \"node\");", + "pm.test(\"three adapter identities, each with a signing key\", () => {", + " pm.expect(nodes.length).to.be.at.least(3);", + " nodes.forEach(n => pm.expect((n.keys || []).length, n.participantId).to.be.above(0));", + "});", + "", + "pm.test(\"both upstreams are registered as type upstream\", () => {", + " [pm.variables.get(\"providerId\"), pm.variables.get(\"mandiProviderId\")].forEach(id => {", + " const up = rows.filter(r => r.participantId === id);", + " pm.expect(up.length, \"no row for \" + id).to.eql(1);", + " pm.expect(up[0].type, id).to.eql(\"upstream\");", + " });", + "});" + ] + } } - } + ] }, - "description": "Search takes NO token -- it is the one registry call a network peer actually needs. That is also why the dev deployment keeps the whole service off the public edge: SunbirdRC uses POST for both reads and writes, so no method rule separates this from a create.\n\nAn empty filters object returns everything. Narrow it with e.g. {\"filters\":{\"participantId\":{\"eq\":\"exp.oan.dev\"}}}." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const rows = pm.response.json().data;", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "", - "// The three adapter identities must exist WITH keys, or nothing can sign.", - "const nodes = rows.filter(r => r.type === \"node\");", - "pm.test(\"three adapter identities, each with a signing key\", () => {", - " pm.expect(nodes.length).to.be.at.least(3);", - " nodes.forEach(n => pm.expect((n.keys || []).length, n.participantId).to.be.above(0));", - "});", - "", - "pm.test(\"both upstreams are registered as type upstream\", () => {", - " [pm.variables.get(\"providerId\"), pm.variables.get(\"mandiProviderId\")].forEach(id => {", - " const up = rows.filter(r => r.participantId === id);", - " pm.expect(up.length, \"no row for \" + id).to.eql(1);", - " pm.expect(up[0].type, id).to.eql(\"upstream\");", - " });", - "});" - ] - } - } - ] - }, - { - "name": "13. Registry \u2014 search provider bindings", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": "{{registryUrl}}/ProviderSchema/search", - "body": { - "mode": "raw", - "raw": "{\n \"filters\": {}\n}", - "options": { - "raw": { - "language": "json" + "name": "13. Search provider bindings", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{registryUrl}}/ProviderSchema/search", + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "The call plans, one row per capability. The two differ in path and mapping URL and point at different upstreams -- which is what lets one provider adapter serve both without knowing anything about either." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const rows = pm.response.json().data;", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const keys = rows.map(r => r.bindingKey);", + "", + "pm.test(\"both capability bindings exist\", () => {", + " pm.expect(keys).to.include(pm.variables.get(\"weatherBindingKey\"));", + " pm.expect(keys).to.include(pm.variables.get(\"mandiBindingKey\"));", + "});", + "", + "// The call plans differ, which is the point.", + "pm.test(\"each binding carries its own path\", () => {", + " const paths = {};", + " rows.forEach(r => { paths[r.bindingKey] = (r.actions || [])[0] && r.actions[0].path; });", + " pm.expect(paths[pm.variables.get(\"weatherBindingKey\")]).to.eql(pm.variables.get(\"weatherPath\"));", + " pm.expect(paths[pm.variables.get(\"mandiBindingKey\")]).to.eql(pm.variables.get(\"mandiPath\"));", + "});" + ] + } } - } - }, - "description": "The call plans, one row per capability. The two differ in path and mapping URL and point at different upstreams -- which is what lets one provider adapter serve both without knowing anything about either." - }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const rows = pm.response.json().data;", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const keys = rows.map(r => r.bindingKey);", - "", - "pm.test(\"both capability bindings exist\", () => {", - " pm.expect(keys).to.include(pm.variables.get(\"weatherBindingKey\"));", - " pm.expect(keys).to.include(pm.variables.get(\"mandiBindingKey\"));", - "});", - "", - "// The call plans differ, which is the point.", - "pm.test(\"each binding carries its own path\", () => {", - " const paths = {};", - " rows.forEach(r => { paths[r.bindingKey] = (r.actions || [])[0] && r.actions[0].path; });", - " pm.expect(paths[pm.variables.get(\"weatherBindingKey\")]).to.eql(pm.variables.get(\"weatherPath\"));", - " pm.expect(paths[pm.variables.get(\"mandiBindingKey\")]).to.eql(pm.variables.get(\"mandiPath\"));", - "});" - ] - } + ] } - ] + ], + "description": "Everything that talks to the registry: one write token, the records bin/setup.py already created, updates for the fields that change, and the two searches.\n\nRUN THE TOKEN FIRST. Requests 2-11 need it; it is saved to {{token}}. Searches need no token at all -- that is the one registry call a network peer makes, and the reason the whole service is kept off the public edge, since SunbirdRC uses POST for reads and writes alike.\n\nNone of this changes a seeded stack. The creates report \"already present\" because the registry is append-only, and the updates write back the value the variable already holds." }, { - "name": "14. Publish \u2014 weather catalogue", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": "{{providerAdapterUrl}}/publish", - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" + "name": "2. Publish", + "item": [ + { + "name": "1. Weather catalogue", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{providerAdapterUrl}}/publish", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Enters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service. Posting straight at the discovery service would skip both adapters, and so skip the part worth testing.\n\ncontext.action is catalog/publish, the name the Beckn v2 spec gives this action at /catalog/publish. A bare \"publish\" is not a spec action. The callback is catalog/on_publish.\n\nThe caller signs nothing and the body names no party: identity travels in the Authorization header's keyId, from the adapter's own keyManager config.\n\nThe catalogue carries no offers -- nothing requires them, and select does not read them: it carries its own offer in the request." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", + "pm.test(\"ACCEPTED\", () => pm.expect(b.message.results[0].status).to.eql(\"ACCEPTED\"));" + ] + } } - } + ] }, - "description": "Enters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service. Posting straight at the discovery service would skip both adapters, and so skip the part worth testing.\n\ncontext.action is catalog/publish, the name the Beckn v2 spec gives this action at /catalog/publish. A bare \"publish\" is not a spec action. The callback is catalog/on_publish.\n\nThe caller signs nothing and the body names no party: identity travels in the Authorization header's keyId, from the adapter's own keyManager config.\n\nThe catalogue carries no offers -- nothing requires them, and select does not read them: it carries its own offer in the request." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const b = pm.response.json();", - "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", - "pm.test(\"ACCEPTED\", () => pm.expect(b.message.results[0].status).to.eql(\"ACCEPTED\"));" - ] - } - } - ] - }, - { - "name": "15. Publish \u2014 mandi catalogue", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": "{{providerAdapterUrl}}/publish", - "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8.", - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", - "options": { - "raw": { - "language": "json" + "name": "2. Mandi catalogue", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{providerAdapterUrl}}/publish", + "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8.", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } } - } - } - }, - "response": [], - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const b = pm.response.json();", - "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", - "const r = b.message.results[0];", - "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", - "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(\"cat-agmarknet-mandi-prices\"));" - ] - } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", + "const r = b.message.results[0];", + "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", + "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(\"cat-agmarknet-mandi-prices\"));" + ] + } + } + ] } - ] + ], + "description": "A provider's catalogue entering the network.\n\nThese post to the PROVIDER adapter, which signs on the caller's behalf and forwards to the network layer -- so the body carries no signature and names no party. That module has no signature check at all, which is why the gateway denies /publish from outside.\n\nA provider that holds its own signing key can skip all of that and post straight at the network adapter, which verifies against its registry row. See the keys block on the upstream creates in the Registry folder.\n\nRUN THESE BEFORE DISCOVER. They seed the catalogues it searches for." }, { - "name": "16. Discover \u2014 weather", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{expAdapterUrl}}/discover", - "host": [ - "{{expAdapterUrl}}/discover" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" + "name": "3. Discover", + "item": [ + { + "name": "1. Weather", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{expAdapterUrl}}/discover", + "host": [ + "{{expAdapterUrl}}/discover" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", + "pm.test(\"at least one catalogue\", () => pm.expect((b.message.catalogs || []).length).to.be.above(0));" + ] + } } - } + ] }, - "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const b = pm.response.json();", - "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", - "pm.test(\"at least one catalogue\", () => pm.expect((b.message.catalogs || []).length).to.be.above(0));" - ] - } - } - ] - }, - { - "name": "17. Discover \u2014 mandi", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": "{{expAdapterUrl}}/discover", - "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing.", - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" + "name": "2. Mandi", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{expAdapterUrl}}/discover", + "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing.", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } } - } - } - }, - "response": [], - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const b = pm.response.json();", - "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", - "const cats = b.message.catalogs || [];", - "pm.test(\"at least one catalogue\", () => pm.expect(cats.length).to.be.above(0));", - "// The mandi catalogue specifically, so this cannot pass on the weather one.", - "pm.test(\"the mandi catalogue is discoverable\", () => {", - " pm.expect(cats.map(c => c.id)).to.include(\"cat-agmarknet-mandi-prices\");", - "});", - "// A catalogue entry advertises a capability rather than carrying data.", - "pm.test(\"it advertises OnDemand and carries no prices\", () => {", - " const mandi = cats.find(c => c.id === \"cat-agmarknet-mandi-prices\");", - " const ra = mandi.resources[0].resourceAttributes;", - " pm.expect(ra.informationMode).to.eql(\"OnDemand\");", - " pm.expect(ra).to.not.have.property(\"prices\");", - " pm.expect(ra.supportedCommodities.length).to.be.above(0);", - "});" - ] - } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", + "const cats = b.message.catalogs || [];", + "pm.test(\"at least one catalogue\", () => pm.expect(cats.length).to.be.above(0));", + "// The mandi catalogue specifically, so this cannot pass on the weather one.", + "pm.test(\"the mandi catalogue is discoverable\", () => {", + " pm.expect(cats.map(c => c.id)).to.include(\"cat-agmarknet-mandi-prices\");", + "});", + "// A catalogue entry advertises a capability rather than carrying data.", + "pm.test(\"it advertises OnDemand and carries no prices\", () => {", + " const mandi = cats.find(c => c.id === \"cat-agmarknet-mandi-prices\");", + " const ra = mandi.resources[0].resourceAttributes;", + " pm.expect(ra.informationMode).to.eql(\"OnDemand\");", + " pm.expect(ra).to.not.have.property(\"prices\");", + " pm.expect(ra.supportedCommodities.length).to.be.above(0);", + "});" + ] + } + } + ] } - ] + ], + "description": "Catalogue search, through the experience adapter to the network layer and on to the discovery service.\n\nScoped by networkId: a catalogue published under a different one is invisible here, and comes back as an empty result rather than an error. If discover returns nothing, check {{networkId}} against what the publish requests used before looking anywhere else." }, { - "name": "18. Select \u2014 weather, per-day forecast", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": { - "raw": "{{expAdapterUrl}}/select", - "host": [ - "{{expAdapterUrl}}/select" - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" + "name": "4. Select", + "item": [ + { + "name": "1. Weather, per-day forecast", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": { + "raw": "{{expAdapterUrl}}/select", + "host": [ + "{{expAdapterUrl}}/select" + ] + }, + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code === 404) {", + " console.log(\"404: the provider adapter serves no capability matching this payload.\",", + " \"offer.provider.id and resourceAttributes.@type must equal the deployed\",", + " \"binding key -- check request 2 against PROVIDER_PARTICIPANT_ID in .env\");", + "}", + "", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", + "const c = b.message.contract.commitments[0];", + "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", + "", + "// Spec conformance of the answer, which is the mapping's job and so the", + "// thing that silently regresses when the published mapping changes.", + "pm.test(\"status is in the spec enum\", () => {", + " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", + "});", + "pm.test(\"every resource carries a quantity\", () => {", + " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", + "});", + "pm.test(\"the offer references only resources returned\", () => {", + " const ids = c.resources.map(r => r.id);", + " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", + "});", + "pm.test(\"no party named in the answer\", () => {", + " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", + "});" + ] + } } - } + ] }, - "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." - }, - "event": [ { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "if (pm.response.code === 404) {", - " console.log(\"404: the provider adapter serves no capability matching this payload.\",", - " \"offer.provider.id and resourceAttributes.@type must equal the deployed\",", - " \"binding key -- check request 2 against PROVIDER_PARTICIPANT_ID in .env\");", - "}", - "", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const b = pm.response.json();", - "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", - "const c = b.message.contract.commitments[0];", - "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", - "", - "// Spec conformance of the answer, which is the mapping's job and so the", - "// thing that silently regresses when the published mapping changes.", - "pm.test(\"status is in the spec enum\", () => {", - " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", - "});", - "pm.test(\"every resource carries a quantity\", () => {", - " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", - "});", - "pm.test(\"the offer references only resources returned\", () => {", - " const ids = c.resources.map(r => r.id);", - " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", - "});", - "pm.test(\"no party named in the answer\", () => {", - " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", - "});" - ] - } - } - ] - }, - { - "name": "19. Select \u2014 mandi, prices per market day", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "url": "{{expAdapterUrl}}/select", - "description": "The same endpoint as request 5, the same adapter, a different capability. Nothing routes this: the payload's provider id and resourceAttributes @type form a binding key, the mandi step recognises it and the weather step passes it through.\n\nThe answer is a Direct openagrinet:MandiPrice per price record. Its prices arrive from the upstream as STRINGS with Title Case keys containing spaces, so the mapping converts them; and a record that reported no minimum or maximum must come back with those absent rather than zeroed, which is what the last assertion checks.\n\nThe upstream's credential is a query parameter, which the adapter adds from an environment variable and redacts from the URL it logs.", - "body": { - "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" + "name": "2. Mandi, prices per market day", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{expAdapterUrl}}/select", + "description": "The same endpoint as request 5, the same adapter, a different capability. Nothing routes this: the payload's provider id and resourceAttributes @type form a binding key, the mandi step recognises it and the weather step passes it through.\n\nThe answer is a Direct openagrinet:MandiPrice per price record. Its prices arrive from the upstream as STRINGS with Title Case keys containing spaces, so the mapping converts them; and a record that reported no minimum or maximum must come back with those absent rather than zeroed, which is what the last assertion checks.\n\nThe upstream's credential is a query parameter, which the adapter adds from an environment variable and redacts from the URL it logs.", + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } } - } - } - }, - "response": [], - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "if (pm.response.code === 404) {", - " console.log(\"404: no step matched. offer.provider.id and resourceAttributes.@type must\",", - " \"equal a configured binding key -- check MANDI_PARTICIPANT_ID in .env and\",", - " \"that mandi is in the provider adapter's steps: list, not just providerSteps\");", - "}", - "", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const b = pm.response.json();", - "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", - "const c = b.message.contract.commitments[0];", - "pm.test(\"a resource per price record\", () => pm.expect(c.resources.length).to.be.above(0));", - "", - "const first = c.resources[0].resourceAttributes;", - "pm.test(\"MandiPrice in Direct mode\", () => {", - " pm.expect(first[\"@type\"]).to.eql(\"openagrinet:MandiPrice\");", - " pm.expect(first.informationMode).to.eql(\"Direct\");", - "});", - "// Direct requires all six of these in the pack.", - "pm.test(\"the pack's Direct fields are all present\", () => {", - " [\"source\",\"commodity\",\"market\",\"arrivalDate\",\"prices\",\"generatedAt\"].forEach(", - " f => pm.expect(first, f).to.have.property(f));", - "});", - "// The upstream sends prices as strings; the pack requires numbers.", - "pm.test(\"prices are numbers, not strings\", () => {", - " pm.expect(first.prices.modal).to.be.a(\"number\");", - " pm.expect(first.prices.currency).to.eql(\"INR\");", - "});", - "// dd-MM-yyyy upstream, ISO in the answer.", - "pm.test(\"arrivalDate is ISO\", () => {", - " pm.expect(first.arrivalDate).to.match(/^\\d{4}-\\d{2}-\\d{2}$/);", - "});", - "pm.test(\"status is in the spec enum\", () => {", - " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", - "});", - "pm.test(\"every resource carries a quantity\", () => {", - " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", - "});", - "// A record the market reported partially must come back partial, not zeroed:", - "// \"no minimum reported\" and \"a minimum of zero\" are different facts.", - "pm.test(\"an unreported price is absent, not zero\", () => {", - " const last = c.resources[c.resources.length - 1].resourceAttributes.prices;", - " if (c.resources.length > 1) {", - " pm.expect(last).to.not.have.property(\"minimum\");", - " pm.expect(last).to.not.have.property(\"maximum\");", - " }", - " pm.expect(last.modal).to.be.a(\"number\");", - "});" - ] - } + }, + "response": [], + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if (pm.response.code === 404) {", + " console.log(\"404: no step matched. offer.provider.id and resourceAttributes.@type must\",", + " \"equal a configured binding key -- check MANDI_PARTICIPANT_ID in .env and\",", + " \"that mandi is in the provider adapter's steps: list, not just providerSteps\");", + "}", + "", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "const b = pm.response.json();", + "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", + "const c = b.message.contract.commitments[0];", + "pm.test(\"a resource per price record\", () => pm.expect(c.resources.length).to.be.above(0));", + "", + "const first = c.resources[0].resourceAttributes;", + "pm.test(\"MandiPrice in Direct mode\", () => {", + " pm.expect(first[\"@type\"]).to.eql(\"openagrinet:MandiPrice\");", + " pm.expect(first.informationMode).to.eql(\"Direct\");", + "});", + "// Direct requires all six of these in the pack.", + "pm.test(\"the pack's Direct fields are all present\", () => {", + " [\"source\",\"commodity\",\"market\",\"arrivalDate\",\"prices\",\"generatedAt\"].forEach(", + " f => pm.expect(first, f).to.have.property(f));", + "});", + "// The upstream sends prices as strings; the pack requires numbers.", + "pm.test(\"prices are numbers, not strings\", () => {", + " pm.expect(first.prices.modal).to.be.a(\"number\");", + " pm.expect(first.prices.currency).to.eql(\"INR\");", + "});", + "// dd-MM-yyyy upstream, ISO in the answer.", + "pm.test(\"arrivalDate is ISO\", () => {", + " pm.expect(first.arrivalDate).to.match(/^\\d{4}-\\d{2}-\\d{2}$/);", + "});", + "pm.test(\"status is in the spec enum\", () => {", + " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", + "});", + "pm.test(\"every resource carries a quantity\", () => {", + " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", + "});", + "// A record the market reported partially must come back partial, not zeroed:", + "// \"no minimum reported\" and \"a minimum of zero\" are different facts.", + "pm.test(\"an unreported price is absent, not zero\", () => {", + " const last = c.resources[c.resources.length - 1].resourceAttributes.prices;", + " if (c.resources.length > 1) {", + " pm.expect(last).to.not.have.property(\"minimum\");", + " pm.expect(last).to.not.have.property(\"maximum\");", + " }", + " pm.expect(last.modal).to.be.a(\"number\");", + "});" + ] + } + } + ] } - ] + ], + "description": "The synchronous one. No callback -- the answer is the HTTP response.\n\nBoth requests hit the SAME endpoint on the same adapter, and different domain packages answer them. Each provider step builds a binding key from the payload -- the provider id plus the capability @type -- serves the request if the key is its own, and passes it through untouched if not. Nothing routes by URL, path or domain, which is what lets one adapter host both capabilities.\n\nA 404 NET_ENTITY_NOT_FOUND here means no step claimed the payload's binding key. Compare it against the bindings in the Registry folder." } ] } diff --git a/docker-deployment/postman-collection/README.md b/docker-deployment/postman-collection/README.md index de26535..57d2ff5 100644 --- a/docker-deployment/postman-collection/README.md +++ b/docker-deployment/postman-collection/README.md @@ -21,18 +21,21 @@ stay intact for the next person. No VM hostname or address is committed in either file. Deployment addresses are shared separately, and the environment file is the place to put them. -## The 19 requests - - 1 get a write token saves {{token}} - 2-4 create the exp / network / provider nodes - 5-8 create both upstreams and both bindings - 9-10 update an upstream's URL (PUT) - 11 update a binding's call plan (PUT) - 12-13 search participants / provider bindings - 14-19 publish, discover, select -- both capabilities - -Run in order the first time: request 1 issues the token that 2-11 need, and the -publish requests seed the catalogues discover looks for. +## Four folders + + 1. Registry 13 requests -- token, creates, updates, searches + 2. Publish 2 -- one catalogue per capability + 3. Discover 2 + 4. Select 2 + +**Their order matters on a first run.** The token in Registry is what the +writes after it use, and Publish seeds the catalogues Discover searches for. So +run it top to bottom once; after that any folder runs on its own: + + newman run OAN-dev-flow.postman_collection.json --folder "4. Select" + +Each folder carries a description explaining what that leg of the flow does and +what its failures mean. ## A default run changes nothing From 831f190af768057ade273851676c5b44d54df2aa Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Sun, 6 Sep 2026 16:59:12 +0530 Subject: [PATCH 44/81] chore: update the Postman collection from Postman [OpenAgriNet/network-adapter#4] Re-exported after editing in the app. Four request bodies change; the folder structure, the Registry and Select folders, every script, variable, URL and header are untouched. Most of the line count is Postman reindenting and reordering keys rather than anything meaningful. What actually changed: Publish, both drop networkId, domain, schemaContext, and the provider's availableAt geo polygon Discover, both drop networkId and domain Verified against a live stack before committing, because dropping networkId is the kind of change that fails quietly: discovery scopes catalogues by it, and a mismatch returns an empty result rather than an error, so a broken discover still looks like a passing request. It does not break: the full run is 22 requests and 50 assertions with no failures, discover finds both catalogues, and both selects answer. Also strips " Copy 2" from the collection name, which the export picked up from being edited as a duplicate. Nothing but the name; it would otherwise show up that way in everyone's Postman. --- .../OAN-dev-flow.postman_collection.json | 886 +++++++++++------- 1 file changed, 532 insertions(+), 354 deletions(-) diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index e9274e2..189a736 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -1,169 +1,31 @@ { "info": { + "_postman_id": "7458e3b4-dd16-4ec1-84ce-006f2e423183", "name": "OAN dev \u2014 registry, publish, discover, select", "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nFOUR FOLDERS, AND THEIR ORDER MATTERS. Registry, Publish, Discover, Select. The token in Registry is what the writes after it use, and Publish seeds the catalogues Discover searches for -- so a first run should go top to bottom. After that any folder runs on its own.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "42114807" }, - "variable": [ - { - "key": "registryUrl", - "value": "http://localhost:8081/api/v1", - "description": "Loopback. Tunnel to the VM if it is not this machine." - }, - { - "key": "keycloakUrl", - "value": "http://localhost:8080", - "description": "Issues the write token." - }, - { - "key": "keycloakRealm", - "value": "sunbird-rc" - }, - { - "key": "keycloakClientId", - "value": "registry-frontend" - }, - { - "key": "registryUser", - "value": "no-user" - }, - { - "key": "registryPassword", - "value": "no-user-password" - }, - { - "key": "token", - "value": "", - "description": "Set by request 1. Do not fill in by hand." - }, - { - "key": "targetOsid", - "value": "", - "description": "Set by the PUT requests' pre-request scripts. Do not fill in by hand." - }, - { - "key": "expAdapterUrl", - "value": "http://localhost:9202", - "description": "Takes unsigned requests -- the app is inside the trust boundary." - }, - { - "key": "providerAdapterUrl", - "value": "http://localhost:9200", - "description": "Where publish enters." - }, - { - "key": "networkAdapterUrl", - "value": "http://localhost:9201", - "description": "The network layer adapter -- the peer-facing surface. No request in this collection uses it: discover reaches it via the experience adapter, and publish via the provider adapter. It is here because it is the one adapter besides exp that a deployment exposes publicly, and because its /publish and /discover both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not scripted here." - }, - { - "key": "discoveryUrl", - "value": "http://localhost:8090" - }, - { - "key": "expNodeId", - "value": "exp.oan.dev" - }, - { - "key": "expNodeKey", - "value": "", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "networkNodeId", - "value": "network.oan.dev" - }, - { - "key": "networkNodeKey", - "value": "", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "providerNodeId", - "value": "provider.oan.dev" - }, - { - "key": "providerNodeKey", - "value": "", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "providerId", - "value": "mausamgram-mock", - "description": "The weather upstream. Half of its binding key." - }, - { - "key": "weatherCapability", - "value": "openagrinet:WeatherObservation" - }, - { - "key": "weatherBindingKey", - "value": "mausamgram-mock|openagrinet:WeatherObservation", - "description": "Used to look up the binding's osid for the update." - }, - { - "key": "weatherProviderKey", - "value": "", - "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the weather provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." - }, - { - "key": "weatherBaseUrl", - "value": "http://mockimd:9100", - "description": "A compose service name: reached from inside the network." - }, - { - "key": "weatherPath", - "value": "/get-daily" - }, - { - "key": "weatherMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml" - }, - { - "key": "mandiProviderId", - "value": "agmarknet-mock", - "description": "The mandi upstream. Half of its binding key." - }, - { - "key": "mandiCapability", - "value": "openagrinet:MandiPrice" - }, - { - "key": "mandiBindingKey", - "value": "agmarknet-mock|openagrinet:MandiPrice" - }, - { - "key": "mandiProviderKey", - "value": "", - "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the mandi provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." - }, - { - "key": "mandiBaseUrl", - "value": "http://mockagmarknet:9101" - }, - { - "key": "mandiPath", - "value": "/v1/fetch-agmarknet-vistaar" - }, - { - "key": "mandiMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml" - }, - { - "key": "networkId", - "value": "oan-dev" - }, - { - "key": "domain", - "value": "oan-dev" - } - ], "item": [ { "name": "1. Registry", "item": [ { "name": "1. Get a write token", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const j = pm.response.json();", + "pm.test(\"200\", () => pm.response.to.have.status(200));", + "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", + "pm.collectionVariables.set(\"token\", j.access_token);" + ] + } + } + ], "request": { "method": "POST", "header": [ @@ -176,7 +38,6 @@ "value": "http" } ], - "url": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", "body": { "mode": "urlencoded", "urlencoded": [ @@ -198,49 +59,26 @@ } ] }, + "url": { + "raw": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", + "host": [ + "{{keycloakUrl}}" + ], + "path": [ + "auth", + "realms", + "{{keycloakRealm}}", + "protocol", + "openid-connect", + "token" + ] + }, "description": "Every registry WRITE needs this. Searches take no token at all.\n\nTHE TWO X-Forwarded-* HEADERS ARE NOT OPTIONAL, and keycloak:8080 is the container-internal address on purpose -- not whatever port Keycloak is published on. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and every write below returns 401 with an empty body.\n\nSaved to {{token}}, so run this first.\n\nA 500 here usually means Keycloak's realm is missing -- it shares a database with the registry, so wiping the registry volume takes the realm with it. Restarting Keycloak re-imports it." }, - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const j = pm.response.json();", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", - "pm.collectionVariables.set(\"token\", j.access_token);" - ] - } - } - ] + "response": [] }, { "name": "2. Create the exp node", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "url": "{{registryUrl}}/Participant", - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{expNodeId}}\",\n \"name\": \"OAN experience layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{expNodeId}}\",\n \"role\": \"consumer\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{expNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, "event": [ { "listen": "test", @@ -272,10 +110,7 @@ ] } } - ] - }, - { - "name": "3. Create the network node", + ], "request": { "method": "POST", "header": [ @@ -288,18 +123,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant", "body": { "mode": "raw", - "raw": "{\n \"participantId\": \"{{networkNodeId}}\",\n \"name\": \"OAN network layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{networkNodeId}}\",\n \"role\": \"network\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{networkNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "raw": "{\n \"participantId\": \"{{expNodeId}}\",\n \"name\": \"OAN experience layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{expNodeId}}\",\n \"role\": \"consumer\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{expNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, + "url": { + "raw": "{{registryUrl}}/Participant", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant" + ] + }, "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." }, + "response": [] + }, + { + "name": "3. Create the network node", "event": [ { "listen": "test", @@ -331,10 +178,7 @@ ] } } - ] - }, - { - "name": "4. Create the provider node", + ], "request": { "method": "POST", "header": [ @@ -347,18 +191,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant", "body": { "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerNodeId}}\",\n \"name\": \"OAN provider layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{providerNodeId}}\",\n \"role\": \"provider\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{providerNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "raw": "{\n \"participantId\": \"{{networkNodeId}}\",\n \"name\": \"OAN network layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{networkNodeId}}\",\n \"role\": \"network\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{networkNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, + "url": { + "raw": "{{registryUrl}}/Participant", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant" + ] + }, "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." }, + "response": [] + }, + { + "name": "4. Create the provider node", "event": [ { "listen": "test", @@ -390,10 +246,7 @@ ] } } - ] - }, - { - "name": "5. Create the weather upstream", + ], "request": { "method": "POST", "header": [ @@ -406,18 +259,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant", "body": { "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{weatherProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "raw": "{\n \"participantId\": \"{{providerNodeId}}\",\n \"name\": \"OAN provider layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{providerNodeId}}\",\n \"role\": \"provider\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{providerNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." + "url": { + "raw": "{{registryUrl}}/Participant", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant" + ] + }, + "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." }, + "response": [] + }, + { + "name": "5. Create the weather upstream", "event": [ { "listen": "prerequest", @@ -482,10 +347,7 @@ ] } } - ] - }, - { - "name": "6. Create the weather capability binding", + ], "request": { "method": "POST", "header": [ @@ -498,18 +360,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/ProviderSchema", "body": { "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{providerId}}|{{weatherCapability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{weatherCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{weatherProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." + "url": { + "raw": "{{registryUrl}}/Participant", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant" + ] + }, + "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." }, + "response": [] + }, + { + "name": "6. Create the weather capability binding", "event": [ { "listen": "test", @@ -539,10 +413,7 @@ ] } } - ] - }, - { - "name": "7. Create the mandi upstream", + ], "request": { "method": "POST", "header": [ @@ -555,18 +426,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant", "body": { "mode": "raw", - "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{mandiProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", + "raw": "{\n \"bindingKey\": \"{{providerId}}|{{weatherCapability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{weatherCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." + "url": { + "raw": "{{registryUrl}}/ProviderSchema", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "ProviderSchema" + ] + }, + "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." }, + "response": [] + }, + { + "name": "7. Create the mandi upstream", "event": [ { "listen": "prerequest", @@ -631,10 +514,7 @@ ] } } - ] - }, - { - "name": "8. Create the mandi capability binding", + ], "request": { "method": "POST", "header": [ @@ -647,18 +527,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/ProviderSchema", "body": { "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{mandiProviderId}}|{{mandiCapability}}\",\n \"participantId\": \"{{mandiProviderId}}\",\n \"capabilityCode\": \"{{mandiCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{mandiPath}}\",\n \"mappings\": \"{{mandiMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{mandiProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." + "url": { + "raw": "{{registryUrl}}/Participant", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant" + ] + }, + "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." }, + "response": [] + }, + { + "name": "8. Create the mandi capability binding", "event": [ { "listen": "test", @@ -688,12 +580,9 @@ ] } } - ] - }, - { - "name": "9. Update the weather upstream URL (PUT)", + ], "request": { - "method": "PUT", + "method": "POST", "header": [ { "key": "Content-Type", @@ -704,18 +593,30 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant/{{targetOsid}}", "body": { "mode": "raw", - "raw": "{\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", + "raw": "{\n \"bindingKey\": \"{{mandiProviderId}}|{{mandiCapability}}\",\n \"participantId\": \"{{mandiProviderId}}\",\n \"capabilityCode\": \"{{mandiCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{mandiPath}}\",\n \"mappings\": \"{{mandiMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." + "url": { + "raw": "{{registryUrl}}/ProviderSchema", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "ProviderSchema" + ] + }, + "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." }, + "response": [] + }, + { + "name": "9. Update the weather upstream URL (PUT)", "event": [ { "listen": "prerequest", @@ -772,10 +673,7 @@ ] } } - ] - }, - { - "name": "10. Update the mandi upstream URL (PUT)", + ], "request": { "method": "PUT", "header": [ @@ -788,18 +686,31 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant/{{targetOsid}}", "body": { "mode": "raw", - "raw": "{\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", + "raw": "{\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", "options": { "raw": { "language": "json" } } }, + "url": { + "raw": "{{registryUrl}}/Participant/{{targetOsid}}", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant", + "{{targetOsid}}" + ] + }, "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." }, + "response": [] + }, + { + "name": "10. Update the mandi upstream URL (PUT)", "event": [ { "listen": "prerequest", @@ -856,10 +767,7 @@ ] } } - ] - }, - { - "name": "11. Update the weather binding's call plan (PUT)", + ], "request": { "method": "PUT", "header": [ @@ -872,18 +780,31 @@ "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/ProviderSchema/{{targetOsid}}", "body": { "mode": "raw", - "raw": "{\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", + "raw": "{\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", "options": { "raw": { "language": "json" } } }, - "description": "Change a call plan without recreating the binding: a new mapping URL, a different path, a longer timeout, more retries.\n\nUnlike the upstream update this DOES send the whole actions list, because actions is a list and replacing one entry means sending the list. So it also needs additionalProperties: true on ActionBinding, not just on ProviderSchema.\n\nRe-runnable for the same reason: it writes back what the variables already hold." + "url": { + "raw": "{{registryUrl}}/Participant/{{targetOsid}}", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant", + "{{targetOsid}}" + ] + }, + "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." }, + "response": [] + }, + { + "name": "11. Update the weather binding's call plan (PUT)", "event": [ { "listen": "prerequest", @@ -940,30 +861,44 @@ ] } } - ] - }, - { - "name": "12. Search participants", + ], "request": { - "method": "POST", + "method": "PUT", "header": [ { "key": "Content-Type", "value": "application/json" + }, + { + "key": "Authorization", + "value": "Bearer {{token}}" } ], - "url": "{{registryUrl}}/Participant/search", "body": { "mode": "raw", - "raw": "{\n \"filters\": {}\n}", + "raw": "{\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", "options": { "raw": { "language": "json" } } }, - "description": "Search takes NO token -- it is the one registry call a network peer actually needs. That is also why the dev deployment keeps the whole service off the public edge: SunbirdRC uses POST for both reads and writes, so no method rule separates this from a create.\n\nAn empty filters object returns everything. Narrow it with e.g. {\"filters\":{\"participantId\":{\"eq\":\"exp.oan.dev\"}}}." + "url": { + "raw": "{{registryUrl}}/ProviderSchema/{{targetOsid}}", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "ProviderSchema", + "{{targetOsid}}" + ] + }, + "description": "Change a call plan without recreating the binding: a new mapping URL, a different path, a longer timeout, more retries.\n\nUnlike the upstream update this DOES send the whole actions list, because actions is a list and replacing one entry means sending the list. So it also needs additionalProperties: true on ActionBinding, not just on ProviderSchema.\n\nRe-runnable for the same reason: it writes back what the variables already hold." }, + "response": [] + }, + { + "name": "12. Search participants", "event": [ { "listen": "test", @@ -990,10 +925,7 @@ ] } } - ] - }, - { - "name": "13. Search provider bindings", + ], "request": { "method": "POST", "header": [ @@ -1002,7 +934,6 @@ "value": "application/json" } ], - "url": "{{registryUrl}}/ProviderSchema/search", "body": { "mode": "raw", "raw": "{\n \"filters\": {}\n}", @@ -1012,8 +943,22 @@ } } }, - "description": "The call plans, one row per capability. The two differ in path and mapping URL and point at different upstreams -- which is what lets one provider adapter serve both without knowing anything about either." + "url": { + "raw": "{{registryUrl}}/Participant/search", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "Participant", + "search" + ] + }, + "description": "Search takes NO token -- it is the one registry call a network peer actually needs. That is also why the dev deployment keeps the whole service off the public edge: SunbirdRC uses POST for both reads and writes, so no method rule separates this from a create.\n\nAn empty filters object returns everything. Narrow it with e.g. {\"filters\":{\"participantId\":{\"eq\":\"exp.oan.dev\"}}}." }, + "response": [] + }, + { + "name": "13. Search provider bindings", "event": [ { "listen": "test", @@ -1039,16 +984,7 @@ ] } } - ] - } - ], - "description": "Everything that talks to the registry: one write token, the records bin/setup.py already created, updates for the fields that change, and the two searches.\n\nRUN THE TOKEN FIRST. Requests 2-11 need it; it is saved to {{token}}. Searches need no token at all -- that is the one registry call a network peer makes, and the reason the whole service is kept off the public edge, since SunbirdRC uses POST for reads and writes alike.\n\nNone of this changes a seeded stack. The creates report \"already present\" because the registry is append-only, and the updates write back the value the variable already holds." - }, - { - "name": "2. Publish", - "item": [ - { - "name": "1. Weather catalogue", + ], "request": { "method": "POST", "header": [ @@ -1057,35 +993,53 @@ "value": "application/json" } ], - "url": "{{providerAdapterUrl}}/publish", "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"filters\": {}\n}", "options": { "raw": { "language": "json" } } }, - "description": "Enters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service. Posting straight at the discovery service would skip both adapters, and so skip the part worth testing.\n\ncontext.action is catalog/publish, the name the Beckn v2 spec gives this action at /catalog/publish. A bare \"publish\" is not a spec action. The callback is catalog/on_publish.\n\nThe caller signs nothing and the body names no party: identity travels in the Authorization header's keyId, from the adapter's own keyManager config.\n\nThe catalogue carries no offers -- nothing requires them, and select does not read them: it carries its own offer in the request." + "url": { + "raw": "{{registryUrl}}/ProviderSchema/search", + "host": [ + "{{registryUrl}}" + ], + "path": [ + "ProviderSchema", + "search" + ] + }, + "description": "The call plans, one row per capability. The two differ in path and mapping URL and point at different upstreams -- which is what lets one provider adapter serve both without knowing anything about either." }, + "response": [] + } + ], + "description": "Everything that talks to the registry: one write token, the records bin/setup.py already created, updates for the fields that change, and the two searches.\n\nRUN THE TOKEN FIRST. Requests 2-11 need it; it is saved to {{token}}. Searches need no token at all -- that is the one registry call a network peer makes, and the reason the whole service is kept off the public edge, since SunbirdRC uses POST for reads and writes alike.\n\nNone of this changes a seeded stack. The creates report \"already present\" because the registry is append-only, and the updates write back the value the variable already holds." + }, + { + "name": "2. Publish", + "item": [ + { + "name": "1. Weather catalogue", "event": [ { "listen": "test", "script": { - "type": "text/javascript", "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", "pm.test(\"ACCEPTED\", () => pm.expect(b.message.results[0].status).to.eql(\"ACCEPTED\"));" - ] + ], + "type": "text/javascript", + "packages": {}, + "requests": {} } } - ] - }, - { - "name": "2. Mandi catalogue", + ], "request": { "method": "POST", "header": [ @@ -1094,24 +1048,34 @@ "value": "application/json" } ], - "url": "{{providerAdapterUrl}}/publish", - "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8.", "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"domain\": \"{{domain}}\",\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ],\n \"networkId\": \"{{networkId}}\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n },\n \"availableAt\": [\n {\n \"geo\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n }\n ]\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } - } + }, + "url": { + "raw": "{{providerAdapterUrl}}/publish", + "host": [ + "{{providerAdapterUrl}}" + ], + "path": [ + "publish" + ] + }, + "description": "Enters at the PROVIDER adapter, which signs it as itself and forwards to the network layer; the network layer verifies that signature and hands it to the discovery service. Posting straight at the discovery service would skip both adapters, and so skip the part worth testing.\n\ncontext.action is catalog/publish, the name the Beckn v2 spec gives this action at /catalog/publish. A bare \"publish\" is not a spec action. The callback is catalog/on_publish.\n\nThe caller signs nothing and the body names no party: identity travels in the Authorization header's keyId, from the adapter's own keyManager config.\n\nThe catalogue carries no offers -- nothing requires them, and select does not read them: it carries its own offer in the request." }, - "response": [], + "response": [] + }, + { + "name": "2. Mandi catalogue", "event": [ { "listen": "test", "script": { - "type": "text/javascript", "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", @@ -1119,19 +1083,13 @@ "const r = b.message.results[0];", "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(\"cat-agmarknet-mandi-prices\"));" - ] + ], + "type": "text/javascript", + "packages": {}, + "requests": {} } } - ] - } - ], - "description": "A provider's catalogue entering the network.\n\nThese post to the PROVIDER adapter, which signs on the caller's behalf and forwards to the network layer -- so the body carries no signature and names no party. That module has no signature check at all, which is why the gateway denies /publish from outside.\n\nA provider that holds its own signing key can skip all of that and post straight at the network adapter, which verifies against its registry row. See the keys block on the upstream creates in the Registry folder.\n\nRUN THESE BEFORE DISCOVER. They seed the catalogues it searches for." - }, - { - "name": "3. Discover", - "item": [ - { - "name": "1. Weather", + ], "request": { "method": "POST", "header": [ @@ -1140,40 +1098,52 @@ "value": "application/json" } ], - "url": { - "raw": "{{expAdapterUrl}}/discover", - "host": [ - "{{expAdapterUrl}}/discover" - ] - }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n \n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" } } }, - "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." + "url": { + "raw": "{{providerAdapterUrl}}/publish", + "host": [ + "{{providerAdapterUrl}}" + ], + "path": [ + "publish" + ] + }, + "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8." }, + "response": [] + } + ], + "description": "A provider's catalogue entering the network.\n\nThese post to the PROVIDER adapter, which signs on the caller's behalf and forwards to the network layer -- so the body carries no signature and names no party. That module has no signature check at all, which is why the gateway denies /publish from outside.\n\nA provider that holds its own signing key can skip all of that and post straight at the network adapter, which verifies against its registry row. See the keys block on the upstream creates in the Registry folder.\n\nRUN THESE BEFORE DISCOVER. They seed the catalogues it searches for." + }, + { + "name": "3. Discover", + "item": [ + { + "name": "1. Weather", "event": [ { "listen": "test", "script": { - "type": "text/javascript", "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", "pm.test(\"at least one catalogue\", () => pm.expect((b.message.catalogs || []).length).to.be.above(0));" - ] + ], + "type": "text/javascript", + "packages": {}, + "requests": {} } } - ] - }, - { - "name": "2. Mandi", + ], "request": { "method": "POST", "header": [ @@ -1182,24 +1152,34 @@ "value": "application/json" } ], - "url": "{{expAdapterUrl}}/discover", - "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing.", "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"domain\": \"{{domain}}\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", "options": { "raw": { "language": "json" } } - } + }, + "url": { + "raw": "{{expAdapterUrl}}/discover", + "host": [ + "{{expAdapterUrl}}" + ], + "path": [ + "discover" + ] + }, + "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." }, - "response": [], + "response": [] + }, + { + "name": "2. Mandi", "event": [ { "listen": "test", "script": { - "type": "text/javascript", "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", @@ -1218,19 +1198,13 @@ " pm.expect(ra).to.not.have.property(\"prices\");", " pm.expect(ra.supportedCommodities.length).to.be.above(0);", "});" - ] + ], + "type": "text/javascript", + "packages": {}, + "requests": {} } } - ] - } - ], - "description": "Catalogue search, through the experience adapter to the network layer and on to the discovery service.\n\nScoped by networkId: a catalogue published under a different one is invisible here, and comes back as an empty result rather than an error. If discover returns nothing, check {{networkId}} against what the publish requests used before looking anywhere else." - }, - { - "name": "4. Select", - "item": [ - { - "name": "1. Weather, per-day forecast", + ], "request": { "method": "POST", "header": [ @@ -1239,23 +1213,36 @@ "value": "application/json" } ], - "url": { - "raw": "{{expAdapterUrl}}/select", - "host": [ - "{{expAdapterUrl}}/select" - ] - }, "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", "options": { "raw": { "language": "json" } } }, - "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." + "url": { + "raw": "{{expAdapterUrl}}/discover", + "host": [ + "{{expAdapterUrl}}" + ], + "path": [ + "discover" + ] + }, + "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing." }, + "response": [] + } + ], + "description": "Catalogue search, through the experience adapter to the network layer and on to the discovery service.\n\nScoped by networkId: a catalogue published under a different one is invisible here, and comes back as an empty result rather than an error. If discover returns nothing, check {{networkId}} against what the publish requests used before looking anywhere else." + }, + { + "name": "4. Select", + "item": [ + { + "name": "1. Weather, per-day forecast", "event": [ { "listen": "test", @@ -1292,10 +1279,7 @@ ] } } - ] - }, - { - "name": "2. Mandi, prices per market day", + ], "request": { "method": "POST", "header": [ @@ -1304,19 +1288,30 @@ "value": "application/json" } ], - "url": "{{expAdapterUrl}}/select", - "description": "The same endpoint as request 5, the same adapter, a different capability. Nothing routes this: the payload's provider id and resourceAttributes @type form a binding key, the mandi step recognises it and the weather step passes it through.\n\nThe answer is a Direct openagrinet:MandiPrice per price record. Its prices arrive from the upstream as STRINGS with Title Case keys containing spaces, so the mapping converts them; and a record that reported no minimum or maximum must come back with those absent rather than zeroed, which is what the last assertion checks.\n\nThe upstream's credential is a query parameter, which the adapter adds from an environment variable and redacts from the URL it logs.", "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" } } - } + }, + "url": { + "raw": "{{expAdapterUrl}}/select", + "host": [ + "{{expAdapterUrl}}" + ], + "path": [ + "select" + ] + }, + "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." }, - "response": [], + "response": [] + }, + { + "name": "2. Mandi, prices per market day", "event": [ { "listen": "test", @@ -1373,10 +1368,193 @@ ] } } - ] + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{expAdapterUrl}}/select", + "host": [ + "{{expAdapterUrl}}" + ], + "path": [ + "select" + ] + }, + "description": "The same endpoint as request 5, the same adapter, a different capability. Nothing routes this: the payload's provider id and resourceAttributes @type form a binding key, the mandi step recognises it and the weather step passes it through.\n\nThe answer is a Direct openagrinet:MandiPrice per price record. Its prices arrive from the upstream as STRINGS with Title Case keys containing spaces, so the mapping converts them; and a record that reported no minimum or maximum must come back with those absent rather than zeroed, which is what the last assertion checks.\n\nThe upstream's credential is a query parameter, which the adapter adds from an environment variable and redacts from the URL it logs." + }, + "response": [] } ], "description": "The synchronous one. No callback -- the answer is the HTTP response.\n\nBoth requests hit the SAME endpoint on the same adapter, and different domain packages answer them. Each provider step builds a binding key from the payload -- the provider id plus the capability @type -- serves the request if the key is its own, and passes it through untouched if not. Nothing routes by URL, path or domain, which is what lets one adapter host both capabilities.\n\nA 404 NET_ENTITY_NOT_FOUND here means no step claimed the payload's binding key. Compare it against the bindings in the Registry folder." } + ], + "variable": [ + { + "key": "registryUrl", + "value": "http://localhost:8081/api/v1", + "description": "Loopback. Tunnel to the VM if it is not this machine." + }, + { + "key": "keycloakUrl", + "value": "http://localhost:8080", + "description": "Issues the write token." + }, + { + "key": "keycloakRealm", + "value": "sunbird-rc" + }, + { + "key": "keycloakClientId", + "value": "registry-frontend" + }, + { + "key": "registryUser", + "value": "no-user" + }, + { + "key": "registryPassword", + "value": "no-user-password" + }, + { + "key": "token", + "value": "", + "description": "Set by request 1. Do not fill in by hand." + }, + { + "key": "targetOsid", + "value": "", + "description": "Set by the PUT requests' pre-request scripts. Do not fill in by hand." + }, + { + "key": "expAdapterUrl", + "value": "http://localhost:9202", + "description": "Takes unsigned requests -- the app is inside the trust boundary." + }, + { + "key": "providerAdapterUrl", + "value": "http://localhost:9200", + "description": "Where publish enters." + }, + { + "key": "networkAdapterUrl", + "value": "http://localhost:9201", + "description": "The network layer adapter -- the peer-facing surface. No request in this collection uses it: discover reaches it via the experience adapter, and publish via the provider adapter. It is here because it is the one adapter besides exp that a deployment exposes publicly, and because its /publish and /discover both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not scripted here." + }, + { + "key": "discoveryUrl", + "value": "http://localhost:8090" + }, + { + "key": "expNodeId", + "value": "exp.oan.dev" + }, + { + "key": "expNodeKey", + "value": "", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "networkNodeId", + "value": "network.oan.dev" + }, + { + "key": "networkNodeKey", + "value": "", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "providerNodeId", + "value": "provider.oan.dev" + }, + { + "key": "providerNodeKey", + "value": "", + "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + }, + { + "key": "providerId", + "value": "mausamgram-mock", + "description": "The weather upstream. Half of its binding key." + }, + { + "key": "weatherCapability", + "value": "openagrinet:WeatherObservation" + }, + { + "key": "weatherBindingKey", + "value": "mausamgram-mock|openagrinet:WeatherObservation", + "description": "Used to look up the binding's osid for the update." + }, + { + "key": "weatherProviderKey", + "value": "", + "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the weather provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." + }, + { + "key": "weatherBaseUrl", + "value": "http://mockimd:9100", + "description": "A compose service name: reached from inside the network." + }, + { + "key": "weatherPath", + "value": "/get-daily" + }, + { + "key": "weatherMappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml" + }, + { + "key": "mandiProviderId", + "value": "agmarknet-mock", + "description": "The mandi upstream. Half of its binding key." + }, + { + "key": "mandiCapability", + "value": "openagrinet:MandiPrice" + }, + { + "key": "mandiBindingKey", + "value": "agmarknet-mock|openagrinet:MandiPrice" + }, + { + "key": "mandiProviderKey", + "value": "", + "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the mandi provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." + }, + { + "key": "mandiBaseUrl", + "value": "http://mockagmarknet:9101" + }, + { + "key": "mandiPath", + "value": "/v1/fetch-agmarknet-vistaar" + }, + { + "key": "mandiMappingUrl", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml" + }, + { + "key": "networkId", + "value": "oan-dev" + }, + { + "key": "domain", + "value": "oan-dev" + } ] } From 0c2e288434063a0c0784421b1cba813adf1ece25 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Sun, 6 Sep 2026 21:29:06 +0530 Subject: [PATCH 45/81] refactor: rename config/gateway to config/reverse-proxy [OpenAgriNet/network-adapter#4] "gateway" is an overloaded word here. Beckn has gateways of its own -- a network role, and one of the values the adapter's own `role:` accepts -- so a directory called gateway sitting next to the adapter configs reads as though it belongs to that concept. It does not: it holds nginx snippets for the reverse proxy that fronts the stack. Contents unchanged, a git mv plus the eleven references that named the old path: the bind mount in docker-compose.yml, five lines in bin/stack.sh, four in README.md, and one inside http_top.conf pointing at its sibling. The compose profile is still called "gateway" and the service is still nginx-proxy-manager. Renaming those changes what people type -- `make gateway`, `--profile gateway` -- so it is left alone rather than bundled in here. Verified: `docker compose config --profile gateway` resolves the bind mount to config/reverse-proxy/npm-custom, the directory is there, and stack.sh still parses. --- docker-deployment/README.md | 10 +++++----- docker-deployment/bin/stack.sh | 10 +++++----- .../{gateway => reverse-proxy}/npm-advanced/exp.conf | 0 .../npm-custom/http_top.conf | 2 +- .../npm-custom/server_proxy.conf | 0 docker-deployment/docker-compose.yml | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) rename docker-deployment/config/{gateway => reverse-proxy}/npm-advanced/exp.conf (100%) rename docker-deployment/config/{gateway => reverse-proxy}/npm-custom/http_top.conf (95%) rename docker-deployment/config/{gateway => reverse-proxy}/npm-custom/server_proxy.conf (100%) diff --git a/docker-deployment/README.md b/docker-deployment/README.md index 303da06..b115d12 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -75,7 +75,7 @@ first use. Do that before creating anything. | Domain | Forward Hostname | Port | Then | |---|---|---|---| -| `exp.oan.example.com` | `exp-adapter` | 9202 | paste `config/gateway/npm-advanced/exp.conf` into **Advanced** | +| `exp.oan.example.com` | `exp-adapter` | 9202 | paste `config/reverse-proxy/npm-advanced/exp.conf` into **Advanced** | | `network.oan.example.com` | `network-adapter` | 9201 | — | | `provider.oan.example.com` | `provider-adapter` | 9200 | — | @@ -120,7 +120,7 @@ signature against the registry; `oanProviderPublish`, on the exact path the provider's own catalogue system inside the trust boundary. A proxy host pointed at `provider-adapter:9200` therefore exposes `/publish` to anyone. NPM's UI offers no way to route a host while withholding one path, so the block lives in -`config/gateway/npm-custom/server_proxy.conf`, which NPM includes in **every** +`config/reverse-proxy/npm-custom/server_proxy.conf`, which NPM includes in **every** proxy host's server block automatically — a mounted file, not a click, and so not something to remember on one host out of three. @@ -145,9 +145,9 @@ Worth being exact about, because the two look alike in the repo: | File | How it applies | |---|---| -| `config/gateway/npm-custom/http_top.conf` | **Automatic.** NPM includes it at the top of its `http` block. Declares the `exp` rate-limit zone and `limit_req_status 429`. | -| `config/gateway/npm-custom/server_proxy.conf` | **Automatic.** Included in every proxy host's server block. Holds the `/publish` deny. | -| `config/gateway/npm-advanced/exp.conf` | **Manual.** Paste into the experience host's Advanced tab. Applies `limit_req` to that host only, since a 10 r/s ceiling on signed peer traffic would throttle for no security gain. | +| `config/reverse-proxy/npm-custom/http_top.conf` | **Automatic.** NPM includes it at the top of its `http` block. Declares the `exp` rate-limit zone and `limit_req_status 429`. | +| `config/reverse-proxy/npm-custom/server_proxy.conf` | **Automatic.** Included in every proxy host's server block. Holds the `/publish` deny. | +| `config/reverse-proxy/npm-advanced/exp.conf` | **Manual.** Paste into the experience host's Advanced tab. Applies `limit_req` to that host only, since a 10 r/s ceiling on signed peer traffic would throttle for no security gain. | The manual one is in a file anyway because NPM's Advanced field is a textarea in a database row: nothing diffs it and nothing reviews it. Keeping the source diff --git a/docker-deployment/bin/stack.sh b/docker-deployment/bin/stack.sh index 7e5b1e0..b1e88bd 100755 --- a/docker-deployment/bin/stack.sh +++ b/docker-deployment/bin/stack.sh @@ -234,7 +234,7 @@ observability() { # `git pull`, with the one thing that otherwise stops it. # -# config/gateway/npm-custom is bind-mounted into NPM, and NPM's s6 init chowns +# config/reverse-proxy/npm-custom is bind-mounted into NPM, and NPM's s6 init chowns # everything under /data/nginx on every start -- so those two files end up # owned by a UID that is not you, and git cannot unlink them to update: # @@ -245,9 +245,9 @@ observability() { # to fix once and for all. Taking the files back before pulling is the whole # workaround, and it belongs in a target rather than in someone's memory. pull() { - step 1 2 "taking back ownership of config/gateway/npm-custom" - if [ -n "$(find config/gateway/npm-custom ! -user "$(id -un)" -print -quit 2>/dev/null)" ]; then - sudo chown -R "$(id -un):$(id -gn)" config/gateway/npm-custom + step 1 2 "taking back ownership of config/reverse-proxy/npm-custom" + if [ -n "$(find config/reverse-proxy/npm-custom ! -user "$(id -un)" -print -quit 2>/dev/null)" ]; then + sudo chown -R "$(id -un):$(id -gn)" config/reverse-proxy/npm-custom info "done -- NPM had chowned them on its last start" else info "already yours, nothing to do" @@ -261,7 +261,7 @@ pull() { make up new services, changed images or .env make restart changed adapter or registry config - make restart-edge changed config/gateway/npm-custom + make restart-edge changed config/reverse-proxy/npm-custom NEXT } diff --git a/docker-deployment/config/gateway/npm-advanced/exp.conf b/docker-deployment/config/reverse-proxy/npm-advanced/exp.conf similarity index 100% rename from docker-deployment/config/gateway/npm-advanced/exp.conf rename to docker-deployment/config/reverse-proxy/npm-advanced/exp.conf diff --git a/docker-deployment/config/gateway/npm-custom/http_top.conf b/docker-deployment/config/reverse-proxy/npm-custom/http_top.conf similarity index 95% rename from docker-deployment/config/gateway/npm-custom/http_top.conf rename to docker-deployment/config/reverse-proxy/npm-custom/http_top.conf index 33f3651..4ab629a 100644 --- a/docker-deployment/config/gateway/npm-custom/http_top.conf +++ b/docker-deployment/config/reverse-proxy/npm-custom/http_top.conf @@ -9,7 +9,7 @@ # The rate-limit bucket for the experience layer. Declaring the zone costs # nothing until a server block opts in with `limit_req` -- which is a per-host # decision and therefore lives in that host's Advanced tab, not here. See -# config/gateway/npm-advanced/exp.conf. +# config/reverse-proxy/npm-advanced/exp.conf. # # 10 r/s per address with a burst of 20 absorbs a Postman collection run while # still bounding what is, at bottom, an open relay into the network: /exp/ diff --git a/docker-deployment/config/gateway/npm-custom/server_proxy.conf b/docker-deployment/config/reverse-proxy/npm-custom/server_proxy.conf similarity index 100% rename from docker-deployment/config/gateway/npm-custom/server_proxy.conf rename to docker-deployment/config/reverse-proxy/npm-custom/server_proxy.conf diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index ff40d10..e25c2ad 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -575,7 +575,7 @@ services: # What NPM actually rewrites is `listen` directives, and neither file # here has one, so the pass is a no-op on content. It does take # ownership of the files, which is cosmetic and not tracked by git. - - ./config/gateway/npm-custom:/data/nginx/custom + - ./config/reverse-proxy/npm-custom:/data/nginx/custom depends_on: # Ordering only; the adapter image has no healthcheck. From 20f61d9cc591bed50cd3074f1f70a5a14f96513a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Sun, 6 Sep 2026 21:32:46 +0530 Subject: [PATCH 46/81] refactor: rename the gateway profile and target to reverse-proxy [OpenAgriNet/network-adapter#4] Follows the directory rename. "gateway" is a Beckn network role and one of the values the adapter's own `role:` accepts, so using it for the nginx reverse proxy invited exactly the wrong reading -- and leaving `make gateway` starting a profile whose files now live in config/reverse-proxy would have been the same confusion with an extra step. compose profiles: ["gateway"] -> ["reverse-proxy"] and the --profile in the comments stack.sh --profile gateway x4, gateway() -> reverse_proxy(), the dispatch case and the help line Makefile make gateway -> make reverse-proxy, and .PHONY docs README's profile note and layout tree, CERTIFICATES.md THIS CHANGES A COMMAND PEOPLE TYPE. `make gateway` no longer exists; it is `make reverse-proxy`. Nothing aliases the old name -- stack.sh falls through to its usage banner, which names the new one, so the failure explains itself rather than doing nothing. Easy to add an alias if the old name has fingers in it. The SERVICE is still nginx-proxy-manager and the container is still oan-npm. Those are the product's own names, not ours, and renaming them would break `docker compose logs nginx-proxy-manager` for no gain. Verified: compose lists nginx-proxy-manager under --profile reverse-proxy and omits it without, make -n reverse-proxy resolves to the right stack.sh call, stack.sh parses, and nothing in the tree still names the old profile or target. --- docker-deployment/CERTIFICATES.md | 2 +- docker-deployment/Makefile | 6 +++--- docker-deployment/README.md | 4 ++-- docker-deployment/bin/stack.sh | 14 +++++++------- docker-deployment/docker-compose.yml | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker-deployment/CERTIFICATES.md b/docker-deployment/CERTIFICATES.md index eb46661..8e7e3e2 100644 --- a/docker-deployment/CERTIFICATES.md +++ b/docker-deployment/CERTIFICATES.md @@ -73,7 +73,7 @@ the old answer. you do not get to enumerate. Scoping it to yourself fails with a challenge timeout that reads like a DNS problem. -3. `make gateway`, then create the proxy host with `..sslip.io`. +3. `make reverse-proxy`, then create the proxy host with `..sslip.io`. No DNS record to create -- that is the entire point of sslip.io here. 4. Confirm it answers over plain HTTP first. Let's Encrypt allows 5 failed diff --git a/docker-deployment/Makefile b/docker-deployment/Makefile index 9b06361..32b7b8e 100644 --- a/docker-deployment/Makefile +++ b/docker-deployment/Makefile @@ -4,7 +4,7 @@ # thing to type. # # make up the whole stack, in the order it has to start -# make up-core the same minus the gateway and hyperdx +# make up-core the same minus the reverse proxy and hyperdx # make down stop everything, keep the data # make help the rest # @@ -13,7 +13,7 @@ STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh -.PHONY: help up up-core down destroy setup gateway observability pull restart restart-edge ps logs +.PHONY: help up up-core down destroy setup reverse-proxy observability pull restart restart-edge ps logs # Default target: running a bare `make` in a directory that can delete a # Postgres volume should print the menu, not pick something from it. @@ -32,7 +32,7 @@ destroy: ; @$(STACK) destroy # `make up` already starts both. These are for starting one without the other, # or restarting one after a config change. -gateway: ; @$(STACK) gateway +reverse-proxy: ; @$(STACK) reverse-proxy observability: ; @$(STACK) observability # ------------------------------------------------------------------ the rest diff --git a/docker-deployment/README.md b/docker-deployment/README.md index b115d12..6b56dca 100644 --- a/docker-deployment/README.md +++ b/docker-deployment/README.md @@ -28,7 +28,7 @@ Running here: than through the published port. - **gateway** — Nginx Proxy Manager, the only container that publishes on a routable interface. Routes to the three adapters, and issues and renews the - Let's Encrypt certificates from its own UI. Profile `gateway`. + Let's Encrypt certificates from its own UI. Profile `reverse-proxy`. - **hyperdx** — ClickStack: OTLP ingest, ClickHouse, and the UI over it. Profile `observability`. @@ -760,7 +760,7 @@ bin/ Every make target is one line of delegation here. setup.py keys, five registry rows, the adapter configs config/ - gateway/ + reverse-proxy/ npm-custom/ mounted to /data/nginx/custom, which NPM includes http_top.conf on its own: the rate-limit zone declaration, server_proxy.conf and the /publish deny that every proxy host gets diff --git a/docker-deployment/bin/stack.sh b/docker-deployment/bin/stack.sh index b1e88bd..05f7343 100755 --- a/docker-deployment/bin/stack.sh +++ b/docker-deployment/bin/stack.sh @@ -34,7 +34,7 @@ cd "$ROOT" # only acts on services whose profile is active, so a plain `docker compose # down` leaves the gateway and hyperdx containers running and then reports # success. Naming them on teardown is what makes "down" mean down. -PROFILES=(--profile gateway --profile observability) +PROFILES=(--profile reverse-proxy --profile observability) # ------------------------------------------------------------------ output @@ -83,7 +83,7 @@ up() { # 0.0.0.0:80 and :443, deliberately not scoped, because Let's Encrypt # validates HTTP-01 from its own servers. step 4 5 "nginx-proxy-manager -- the public edge (80 and 443, all interfaces)" - docker compose --profile gateway up -d nginx-proxy-manager + docker compose --profile reverse-proxy up -d nginx-proxy-manager # ClickStack. Heaviest thing here by a wide margin: ClickHouse alone wants # 2-4 GB, which is what takes this VM from 8 GB to 16 GB. @@ -209,10 +209,10 @@ WARN # Separate targets rather than part of `up` because neither is needed to # exercise the stack, and hyperdx (ClickHouse) alone wants 2-4 GB. -gateway() { +reverse_proxy() { preflight step 1 1 "nginx-proxy-manager -- publishes 80 and 443 on ALL interfaces" - docker compose --profile gateway up -d nginx-proxy-manager + docker compose --profile reverse-proxy up -d nginx-proxy-manager cat <<'NEXT' The admin UI is on loopback, and ships with a live default login. Tunnel in @@ -306,7 +306,7 @@ restart_app() { # symptom is a bare 502 that looks like the adapter is down. restart_edge() { step 1 1 "restarting nginx-proxy-manager to re-resolve adapter addresses" - docker compose --profile gateway restart nginx-proxy-manager + docker compose --profile reverse-proxy restart nginx-proxy-manager } # Just step 2. Re-run it after editing a .tmpl, or to re-render configs that @@ -327,7 +327,7 @@ bin/stack.sh down stop everything, keep the data destroy stop everything and DELETE every volume setup re-run bin/setup.py only - gateway start nginx-proxy-manager on its own (public, 80/443) + reverse-proxy start nginx-proxy-manager on its own (public, 80/443) observability start hyperdx on its own pull git pull, fixing the npm-custom ownership first restart restart registry, discovery and the adapters only @@ -344,7 +344,7 @@ case "${1:-}" in down) down ;; destroy) destroy ;; setup) setup ;; - gateway) gateway ;; + reverse-proxy) reverse_proxy ;; observability) observability ;; pull) pull ;; restart) restart_app ;; diff --git a/docker-deployment/docker-compose.yml b/docker-deployment/docker-compose.yml index e25c2ad..1e8f520 100644 --- a/docker-deployment/docker-compose.yml +++ b/docker-deployment/docker-compose.yml @@ -21,7 +21,7 @@ # 3. python3 bin/setup.py keys, the three adapter entries, and # the adapter configs they mount # 4. docker compose up -d now the adapters -# 5. docker compose --profile gateway --profile observability up -d +# 5. docker compose --profile reverse-proxy --profile observability up -d # the edge and the telemetry stack # # Naming registry and discovery in step 2 is not tidiness. An adapter config @@ -500,7 +500,7 @@ services: # ========================================================================== # edge -- Nginx Proxy Manager. # - # docker compose --profile gateway up -d + # docker compose --profile reverse-proxy up -d # # NPM rather than a hand-written nginx.conf because it owns the part that is # genuinely tedious to do by hand -- ACME. It requests, installs and renews @@ -514,7 +514,7 @@ services: image: jc21/nginx-proxy-manager:latest container_name: oan-npm restart: unless-stopped - profiles: ["gateway"] + profiles: ["reverse-proxy"] # oan-edge ONLY, and this is the whole security argument for putting a # UI-configured proxy in front of this stack. From 3702ba79122eb27ff2da05d8556f0902ad738f9a Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Sun, 6 Sep 2026 21:40:40 +0530 Subject: [PATCH 47/81] fix(registry): drop the abstract-capability restriction from the schemas [OpenAgriNet/network-adapter#4] Removes the "not" constraint that refused openagrinet:AgricultureCapability and openagrinet:AgricultureResource, in all three places it appeared: ProviderSchema's bindingKey and CapabilityCode, and SchemaRegistry's CapabilityCode. The intent was evidently to stop a provider binding to an abstract base type rather than a concrete capability -- a row that could never be selected, because the adapter dispatches on an exact binding-key match and no payload carries the abstract @type. But that reasoning was nowhere in the schema, so the rule read as an arbitrary denylist of two names, and a denylist of type names in a registry schema is a thing that goes stale as the vocabulary grows. Line-based edit so the rest of the file keeps its formatting: three insertions, six deletions. Rewriting the JSON wholesale reindents everything and buries a one-line change in several hundred. Verified against a live registry after restarting it -- schemas are read only at startup -- by creating a ProviderSchema whose bindingKey and capabilityCode both use AgricultureCapability. Previously refused; now SUCCESSFUL, with neither field mentioned in any error. --- .../config/registry/schemas/ProviderSchema.json | 6 ++---- .../config/registry/schemas/SchemaRegistry.json | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docker-deployment/config/registry/schemas/ProviderSchema.json b/docker-deployment/config/registry/schemas/ProviderSchema.json index 7d25d31..5744c68 100644 --- a/docker-deployment/config/registry/schemas/ProviderSchema.json +++ b/docker-deployment/config/registry/schemas/ProviderSchema.json @@ -15,8 +15,7 @@ "properties": { "bindingKey": { "type": "string", - "pattern": "^[a-z0-9][a-z0-9._:-]{2,63}\\|openagrinet:[A-Z][A-Za-z0-9]*$", - "not": { "pattern": "\\|openagrinet:Agriculture(Capability|Resource)$" } + "pattern": "^[a-z0-9][a-z0-9._:-]{2,63}\\|openagrinet:[A-Z][A-Za-z0-9]*$" }, "participantId": { "$ref": "#/definitions/ParticipantId" }, "capabilityCode": { "$ref": "#/definitions/CapabilityCode" }, @@ -66,8 +65,7 @@ "ParticipantId": { "type": "string", "maxLength": 253, "pattern": "^[a-z0-9][a-z0-9._:-]{2,252}$" }, "CapabilityCode": { "type": "string", - "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$", - "not": { "pattern": "^openagrinet:Agriculture(Capability|Resource)$" } }, + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$" }, "Status": { "type": "string", "enum": ["active", "inactive"] }, "Path": { "type": "string", "maxLength": 512, "description": "Appended to that upstream's baseUrl. Single slashes: an empty segment is never deliberate, and many servers answer //a differently from /a. A trailing slash is allowed, because /api/ and /api are a distinction some APIs make.", diff --git a/docker-deployment/config/registry/schemas/SchemaRegistry.json b/docker-deployment/config/registry/schemas/SchemaRegistry.json index f67b7fe..c398d63 100644 --- a/docker-deployment/config/registry/schemas/SchemaRegistry.json +++ b/docker-deployment/config/registry/schemas/SchemaRegistry.json @@ -22,8 +22,7 @@ }, "CapabilityCode": { "type": "string", - "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$", - "not": { "pattern": "^openagrinet:Agriculture(Capability|Resource)$" } }, + "pattern": "^openagrinet:[A-Z][A-Za-z0-9]*$" }, "Status": { "type": "string", "enum": ["active", "inactive"] } }, From c477fadeb03e25da8093bb07a1b49bfdf265e7cd Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:14:01 +0530 Subject: [PATCH 48/81] refactor: group the collection by capability, not by action [OpenAgriNet/network-adapter#4] Registry, then one folder per capability: 1. Registry token, creates, updates, searches 2. Weather 1. Publish 2. Discover 3. Select 3. Mandi 1. Publish 2. Discover 3. Select Grouping by action put a weather request and a mandi request side by side in Publish, then again in Discover, then again in Select -- so following one capability meant hopping between three folders, and running just one meant picking single requests out of each. By capability, one folder is one capability end to end. `--folder "3. Mandi"` now runs publish, discover and select for mandi and nothing else. Verified each folder standalone: Weather 13 assertions, Mandi 19, Registry 19, no failures. Order is still expressed by position and still matters: Registry issues the token the writes after it need, and inside a folder Publish seeds the catalogue Discover looks for. Full run unchanged at 22 requests and 51 assertions. The two Select requests are now one level apart rather than adjacent, which is worth a note in each folder's description, because they are the pair that shows the dispatch: same endpoint, same adapter, different domain package answering, decided by the payload's binding key. --- .../OAN-dev-flow.postman_collection.json | 178 +++++++++--------- .../postman-collection/README.md | 27 ++- 2 files changed, 103 insertions(+), 102 deletions(-) diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index 189a736..048c4d9 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -2,7 +2,7 @@ "info": { "_postman_id": "7458e3b4-dd16-4ec1-84ce-006f2e423183", "name": "OAN dev \u2014 registry, publish, discover, select", - "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nFOUR FOLDERS, AND THEIR ORDER MATTERS. Registry, Publish, Discover, Select. The token in Registry is what the writes after it use, and Publish seeds the catalogues Discover searches for -- so a first run should go top to bottom. After that any folder runs on its own.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", + "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nTHREE FOLDERS: Registry, then one per capability. Registry issues the token the writes after it need; inside a capability folder Publish seeds the catalogue Discover looks for. So a first run goes top to bottom, and after that any folder -- or any one capability -- runs on its own.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "42114807" }, @@ -1020,10 +1020,10 @@ "description": "Everything that talks to the registry: one write token, the records bin/setup.py already created, updates for the fields that change, and the two searches.\n\nRUN THE TOKEN FIRST. Requests 2-11 need it; it is saved to {{token}}. Searches need no token at all -- that is the one registry call a network peer makes, and the reason the whole service is kept off the public edge, since SunbirdRC uses POST for reads and writes alike.\n\nNone of this changes a seeded stack. The creates report \"already present\" because the registry is append-only, and the updates write back the value the variable already holds." }, { - "name": "2. Publish", + "name": "2. Weather", "item": [ { - "name": "1. Weather catalogue", + "name": "1. Publish", "event": [ { "listen": "test", @@ -1071,7 +1071,7 @@ "response": [] }, { - "name": "2. Mandi catalogue", + "name": "2. Discover", "event": [ { "listen": "test", @@ -1079,10 +1079,8 @@ "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", - "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", - "const r = b.message.results[0];", - "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", - "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(\"cat-agmarknet-mandi-prices\"));" + "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", + "pm.test(\"at least one catalogue\", () => pm.expect((b.message.catalogs || []).length).to.be.above(0));" ], "type": "text/javascript", "packages": {}, @@ -1100,7 +1098,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n \n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", "options": { "raw": { "language": "json" @@ -1108,39 +1106,54 @@ } }, "url": { - "raw": "{{providerAdapterUrl}}/publish", + "raw": "{{expAdapterUrl}}/discover", "host": [ - "{{providerAdapterUrl}}" + "{{expAdapterUrl}}" ], "path": [ - "publish" + "discover" ] }, - "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8." + "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." }, "response": [] - } - ], - "description": "A provider's catalogue entering the network.\n\nThese post to the PROVIDER adapter, which signs on the caller's behalf and forwards to the network layer -- so the body carries no signature and names no party. That module has no signature check at all, which is why the gateway denies /publish from outside.\n\nA provider that holds its own signing key can skip all of that and post straight at the network adapter, which verifies against its registry row. See the keys block on the upstream creates in the Registry folder.\n\nRUN THESE BEFORE DISCOVER. They seed the catalogues it searches for." - }, - { - "name": "3. Discover", - "item": [ + }, { - "name": "1. Weather", + "name": "3. Select \u2014 per-day forecast", "event": [ { "listen": "test", "script": { + "type": "text/javascript", "exec": [ + "if (pm.response.code === 404) {", + " console.log(\"404: the provider adapter serves no capability matching this payload.\",", + " \"offer.provider.id and resourceAttributes.@type must equal the deployed\",", + " \"binding key -- check request 2 against PROVIDER_PARTICIPANT_ID in .env\");", + "}", + "", "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", - "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", - "pm.test(\"at least one catalogue\", () => pm.expect((b.message.catalogs || []).length).to.be.above(0));" - ], - "type": "text/javascript", - "packages": {}, - "requests": {} + "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", + "const c = b.message.contract.commitments[0];", + "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", + "", + "// Spec conformance of the answer, which is the mapping's job and so the", + "// thing that silently regresses when the published mapping changes.", + "pm.test(\"status is in the spec enum\", () => {", + " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", + "});", + "pm.test(\"every resource carries a quantity\", () => {", + " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", + "});", + "pm.test(\"the offer references only resources returned\", () => {", + " const ids = c.resources.map(r => r.id);", + " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", + "});", + "pm.test(\"no party named in the answer\", () => {", + " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", + "});" + ] } } ], @@ -1154,7 +1167,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -1162,20 +1175,26 @@ } }, "url": { - "raw": "{{expAdapterUrl}}/discover", + "raw": "{{expAdapterUrl}}/select", "host": [ "{{expAdapterUrl}}" ], "path": [ - "discover" + "select" ] }, - "description": "Text search across published catalogs.\n\nRoute: **exp adapter \u2192 network adapter \u2192 discovery service.** No provider plugin is involved, and no upstream API is called. The answer comes from the discovery service's own index.\n\nReturns the catalog published in step 3. An empty list almost always means `networkId` or `domain` did not match what was published." + "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." }, "response": [] - }, + } + ], + "description": "The weather capability end to end: publish its catalogue, find it, then ask for a forecast.\n\nRUN IN ORDER the first time -- Publish seeds the catalogue Discover looks for. After that Select works on its own.\n\nSelect is the interesting one. It goes to the same endpoint on the same adapter as Mandi's, and a different domain package answers it: each provider step builds a binding key from the payload, serves the request if the key is its own, and passes it through untouched if not. A 404 NET_ENTITY_NOT_FOUND here means no step claimed this payload -- compare {{providerId}} and the capability @type against the bindings in Registry." + }, + { + "name": "3. Mandi", + "item": [ { - "name": "2. Mandi", + "name": "1. Publish", "event": [ { "listen": "test", @@ -1183,21 +1202,10 @@ "exec": [ "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", - "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", - "const cats = b.message.catalogs || [];", - "pm.test(\"at least one catalogue\", () => pm.expect(cats.length).to.be.above(0));", - "// The mandi catalogue specifically, so this cannot pass on the weather one.", - "pm.test(\"the mandi catalogue is discoverable\", () => {", - " pm.expect(cats.map(c => c.id)).to.include(\"cat-agmarknet-mandi-prices\");", - "});", - "// A catalogue entry advertises a capability rather than carrying data.", - "pm.test(\"it advertises OnDemand and carries no prices\", () => {", - " const mandi = cats.find(c => c.id === \"cat-agmarknet-mandi-prices\");", - " const ra = mandi.resources[0].resourceAttributes;", - " pm.expect(ra.informationMode).to.eql(\"OnDemand\");", - " pm.expect(ra).to.not.have.property(\"prices\");", - " pm.expect(ra.supportedCommodities.length).to.be.above(0);", - "});" + "pm.test(\"catalog/on_publish\", () => pm.expect(b.context.action).to.eql(\"catalog/on_publish\"));", + "const r = b.message.results[0];", + "pm.test(\"ACCEPTED\", () => pm.expect(r.status).to.eql(\"ACCEPTED\"));", + "pm.test(\"the catalogue id comes back\", () => pm.expect(r.catalogId).to.eql(\"cat-agmarknet-mandi-prices\"));" ], "type": "text/javascript", "packages": {}, @@ -1215,7 +1223,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n \n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -1223,60 +1231,46 @@ } }, "url": { - "raw": "{{expAdapterUrl}}/discover", + "raw": "{{providerAdapterUrl}}/publish", "host": [ - "{{expAdapterUrl}}" + "{{providerAdapterUrl}}" ], "path": [ - "discover" + "publish" ] }, - "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing." + "description": "The second catalogue, entering the same way as the first: the provider adapter signs it and the network layer forwards it to discovery. Publishing is capability-agnostic -- no provider step runs, and nothing on this path knows what a MandiPrice is.\n\nThe resource is OnDemand, which is what a catalogue entry should be: it advertises the commodities and price fields this provider CAN answer for. The pack forbids `prices` in that mode, so a catalogue cannot carry stale numbers -- those appear only in the Direct answer to a select, request 8." }, "response": [] - } - ], - "description": "Catalogue search, through the experience adapter to the network layer and on to the discovery service.\n\nScoped by networkId: a catalogue published under a different one is invisible here, and comes back as an empty result rather than an error. If discover returns nothing, check {{networkId}} against what the publish requests used before looking anywhere else." - }, - { - "name": "4. Select", - "item": [ + }, { - "name": "1. Weather, per-day forecast", + "name": "2. Discover", "event": [ { "listen": "test", "script": { - "type": "text/javascript", "exec": [ - "if (pm.response.code === 404) {", - " console.log(\"404: the provider adapter serves no capability matching this payload.\",", - " \"offer.provider.id and resourceAttributes.@type must equal the deployed\",", - " \"binding key -- check request 2 against PROVIDER_PARTICIPANT_ID in .env\");", - "}", - "", "pm.test(\"200\", () => pm.response.to.have.status(200));", "const b = pm.response.json();", - "pm.test(\"on_select\", () => pm.expect(b.context.action).to.eql(\"on_select\"));", - "const c = b.message.contract.commitments[0];", - "pm.test(\"a resource per forecast day\", () => pm.expect(c.resources.length).to.be.above(0));", - "", - "// Spec conformance of the answer, which is the mapping's job and so the", - "// thing that silently regresses when the published mapping changes.", - "pm.test(\"status is in the spec enum\", () => {", - " pm.expect([\"DRAFT\",\"ACTIVE\",\"CLOSED\"]).to.include(c.status.descriptor.code);", - "});", - "pm.test(\"every resource carries a quantity\", () => {", - " c.resources.forEach(r => pm.expect(r, r.id).to.have.property(\"quantity\"));", - "});", - "pm.test(\"the offer references only resources returned\", () => {", - " const ids = c.resources.map(r => r.id);", - " (c.offer.resourceIds || []).forEach(i => pm.expect(ids).to.include(i));", + "pm.test(\"on_discover\", () => pm.expect(b.context.action).to.eql(\"on_discover\"));", + "const cats = b.message.catalogs || [];", + "pm.test(\"at least one catalogue\", () => pm.expect(cats.length).to.be.above(0));", + "// The mandi catalogue specifically, so this cannot pass on the weather one.", + "pm.test(\"the mandi catalogue is discoverable\", () => {", + " pm.expect(cats.map(c => c.id)).to.include(\"cat-agmarknet-mandi-prices\");", "});", - "pm.test(\"no party named in the answer\", () => {", - " [\"bapId\",\"bapUri\",\"bppId\",\"bppUri\"].forEach(f => pm.expect(b.context[f]).to.be.undefined);", + "// A catalogue entry advertises a capability rather than carrying data.", + "pm.test(\"it advertises OnDemand and carries no prices\", () => {", + " const mandi = cats.find(c => c.id === \"cat-agmarknet-mandi-prices\");", + " const ra = mandi.resources[0].resourceAttributes;", + " pm.expect(ra.informationMode).to.eql(\"OnDemand\");", + " pm.expect(ra).to.not.have.property(\"prices\");", + " pm.expect(ra.supportedCommodities.length).to.be.above(0);", "});" - ] + ], + "type": "text/javascript", + "packages": {}, + "requests": {} } } ], @@ -1290,7 +1284,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", "options": { "raw": { "language": "json" @@ -1298,20 +1292,20 @@ } }, "url": { - "raw": "{{expAdapterUrl}}/select", + "raw": "{{expAdapterUrl}}/discover", "host": [ "{{expAdapterUrl}}" ], "path": [ - "select" + "discover" ] }, - "description": "Asks for a priced quote on the resource discover returned.\n\nRoute: **exp adapter \u2192 provider adapter \u2192 mock IMD.** This is where the plugins do their work:\n\n1. `validateSign` verifies the caller's key, fetched from the registry \u2014 the row you saw in step 1\n2. the provider plugin builds the binding key from the payload, asks the registry for the call plan \u2014 the row from step 2 \u2014 resolves the coordinates, calls the upstream, and maps the answer back\n3. `signAck` signs the answer\n\n**The answer is the HTTP response \u2014 there is no callback.** A bare `ACK` here would mean nothing served the request; the adapter now returns `404 NET_ENTITY_NOT_FOUND` in that case rather than pretending to accept it.\n\nThe response quotes **one** resource, carrying the id this request selected, and follows the same schema pack in `informationMode: Direct` \u2014 which requires `observationType`, `source`, `location`, `generatedAt` and `parameters`.\n\n**Two fields here are not in the pack, both deliberately.** The pack carries one validity and one flat `parameters` array per resource, so it cannot express a five-day forecast in the one resource the request selected \u2014 hence `observations`. And its parameter entry is `parameter`/`value`/`unit` only, so `aggregation` is ours, because this provider reports a minimum *and* a maximum for temperature and humidity. Both validate: the pack sets no `additionalProperties`." + "description": "The same route as request 5 -- exp adapter, network adapter, discovery service -- looking for the other catalogue. No provider plugin and no upstream call: discovery answers from its own index.\n\nAn empty list here almost always means networkId or domain did not match what was published, rather than the text search failing." }, "response": [] }, { - "name": "2. Mandi, prices per market day", + "name": "3. Select \u2014 prices per market day", "event": [ { "listen": "test", @@ -1400,7 +1394,7 @@ "response": [] } ], - "description": "The synchronous one. No callback -- the answer is the HTTP response.\n\nBoth requests hit the SAME endpoint on the same adapter, and different domain packages answer them. Each provider step builds a binding key from the payload -- the provider id plus the capability @type -- serves the request if the key is its own, and passes it through untouched if not. Nothing routes by URL, path or domain, which is what lets one adapter host both capabilities.\n\nA 404 NET_ENTITY_NOT_FOUND here means no step claimed the payload's binding key. Compare it against the bindings in the Registry folder." + "description": "The mandi capability end to end: publish its catalogue, find it, then ask for prices.\n\nRUN IN ORDER the first time -- Publish seeds the catalogue Discover looks for.\n\nNothing here is shared with Weather except the adapter, the registry client and the mapper. A different upstream, a different mapping, a different binding key. The mapping is doing more work than weather's: it converts ISO dates to the dd-MM-yyyy Agmarknet wants, sends marketcode only when the request carried one, turns price strings into numbers, and omits a price that was not reported rather than sending a zero." } ], "variable": [ diff --git a/docker-deployment/postman-collection/README.md b/docker-deployment/postman-collection/README.md index 57d2ff5..367aba6 100644 --- a/docker-deployment/postman-collection/README.md +++ b/docker-deployment/postman-collection/README.md @@ -21,21 +21,28 @@ stay intact for the next person. No VM hostname or address is committed in either file. Deployment addresses are shared separately, and the environment file is the place to put them. -## Four folders +## Three folders 1. Registry 13 requests -- token, creates, updates, searches - 2. Publish 2 -- one catalogue per capability - 3. Discover 2 - 4. Select 2 + 2. Weather 1. Publish 2. Discover 3. Select + 3. Mandi 1. Publish 2. Discover 3. Select -**Their order matters on a first run.** The token in Registry is what the -writes after it use, and Publish seeds the catalogues Discover searches for. So -run it top to bottom once; after that any folder runs on its own: +Grouped by capability rather than by action, so one capability is one folder you +can run end to end: - newman run OAN-dev-flow.postman_collection.json --folder "4. Select" + newman run OAN-dev-flow.postman_collection.json --folder "3. Mandi" -Each folder carries a description explaining what that leg of the flow does and -what its failures mean. +**Order is still position.** Registry issues the token the writes after it need, +and inside a capability folder Publish seeds the catalogue Discover looks for. +So a first run goes top to bottom; after that any folder runs on its own. + +The two Select requests are the pair worth comparing. They hit the same endpoint +on the same adapter and different domain packages answer them, because each +provider step recognises its own binding key from the payload and passes through +anything else. Nothing routes by URL, path or domain. + +Each folder carries a description of what that leg does and what its failures +mean. ## A default run changes nothing From 7465939edbbca2a79d8662771cd87915bffebb8b Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:33:58 +0530 Subject: [PATCH 49/81] chore: add chart conventions, lint script, and helm lint CI [#53] Establishes the repo-level scaffolding the charts depend on: naming and structure conventions, a lint script that rebuilds file:// dependencies before linting, and a CI workflow that runs it. .gitignore excludes Helm dependency artifacts, which are regenerated rather than committed. --- .github/workflows/helm-lint.yml | 32 +++++++++ .gitignore | 15 ++++ CONVENTIONS.md | 123 ++++++++++++++++++++++++++++++++ scripts/lint-charts.sh | 87 ++++++++++++++++++++++ 4 files changed, 257 insertions(+) create mode 100644 .github/workflows/helm-lint.yml create mode 100644 .gitignore create mode 100644 CONVENTIONS.md create mode 100755 scripts/lint-charts.sh diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml new file mode 100644 index 0000000..4e4fabe --- /dev/null +++ b/.github/workflows/helm-lint.yml @@ -0,0 +1,32 @@ +name: helm-lint + +on: + pull_request: + paths: + - 'charts/**' + - 'scripts/lint-charts.sh' + - '.github/workflows/helm-lint.yml' + push: + branches: + - main + - development + paths: + - 'charts/**' + - 'scripts/lint-charts.sh' + - '.github/workflows/helm-lint.yml' + +jobs: + lint: + name: lint and render charts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: v3.16.4 + + - name: Lint and render all charts + run: ./scripts/lint-charts.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5874466 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# macOS +.DS_Store + +# Helm dependency artifacts (regenerated by `helm dependency update`) +charts/**/charts/ +charts/**/Chart.lock +*.tgz + +# Rendered output +rendered/ + +# Editor/IDE +.vscode/ +.idea/ +*.swp diff --git a/CONVENTIONS.md b/CONVENTIONS.md new file mode 100644 index 0000000..daff3d6 --- /dev/null +++ b/CONVENTIONS.md @@ -0,0 +1,123 @@ +# Chart conventions + +Rules every chart in this repository follows. They exist so that a service chart +is predictable to read, safe to upgrade, and traceable back to the change that +produced it. + +## Naming + +| Thing | Rule | Example | +|---|---|---| +| Chart directory and `name` | `oan-`, kebab-case, matching the service's repo/deployment name | `oan-registry-service` | +| Library chart | `oan-common` — the only library chart; every service chart depends on it | `oan-common` | +| Reference chart | `oan-template` — copied to start a new chart, never deployed as-is | `oan-template` | +| A deployed component | Named after the **role it plays in OAN**, no `oan-` prefix — the prefix is for the shared library and the starter chart | `registry`, `keycloak` | +| Two charts for the same role | Add the distinguishing implementation as a suffix, only when there is something to distinguish | `postgresql-cnpg` | +| Release name | The service name without the `oan-` prefix, so resources read `registry-service-...` not `oan-registry-service-oan-registry-service` | `helm install registry-service charts/oan-registry-service` | +| Template helpers | Chart-local helpers are namespaced by chart name: `.` | `oan-registry-service.fullname` | +| Value keys | camelCase, matching Kubernetes field names where one exists | `podSecurityContext`, `envFromSecrets` | +| Env var keys in `envConfig` | SCREAMING_SNAKE_CASE | `LOG_LEVEL` | +| Custom labels/annotations | Prefixed with a domain we own | `oan.in/environment: dev` | + +Component charts are named after the role the component plays in OAN, not after +the software that happens to implement it: `registry` and `keycloak`, not +`registry-sunbird-rc` or `keycloak-sunbird-rc`. The implementation is an +implementation detail, and one that can change without the role changing. + +Add an implementation suffix only when it actually distinguishes something — +`postgresql-cnpg` carries `-cnpg` because a plain `postgresql` chart could +reasonably mean several different operators, and which one is in use changes how +the chart is configured and operated. + +The `oan-` prefix is reserved for the shared library (`oan-common`) and the +starter chart (`oan-template`). Every chart, prefixed or not, depends on +`oan-common` and carries the standard OAN labels, including +`app.kubernetes.io/part-of: oan`. + +Chart directory name, `name` in `Chart.yaml`, and the prefix of the chart-local +helpers must all agree. A mismatch is the most common cause of a chart that +lints clean but renders the wrong resource names. + +## Versioning + +Two independent version fields, both required in every `Chart.yaml`: + +- **`version`** — the version of the *chart*, following + [Semantic Versioning](https://semver.org/). Bumped on every chart change, + even a comment-only one. +- **`appVersion`** — the version of the *application image* the chart deploys by + default, quoted. It is the fallback for `image.tag`, so it must be a real, + pullable tag. Bumping it is a chart change, and so requires a `version` bump + too. + +`version` bump rules: + +| Change | Bump | +|---|---| +| New value key with a backward-compatible default; new optional resource | MINOR | +| Bug fix in a template; doc/comment change; `appVersion` bump | PATCH | +| Removing or renaming a value key or helper; changing a default that alters live behaviour; changing an immutable field such as a selector label | MAJOR | + +For `oan-common` specifically: consumers pin `version: "0.1.x"`, so helper +additions ship as PATCH/MINOR and MAJOR is reserved for renaming or changing the +behaviour of an existing helper. A MAJOR bump of `oan-common` means every +consuming chart's pin has to be updated deliberately. + +Pre-1.0.0 charts are still in flux; once a chart is deployed to production it +goes to `1.0.0` and the rules above are binding. + +## Changelog + +Every chart keeps a `CHANGELOG.md` in +[Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. A chart change +is not complete without both the `version` bump and the matching changelog +entry. This is what makes "which chart version introduced this?" answerable. + +## Required in every service chart + +The deployment epic requires these on every component, and `oan-common` +enforces the first two at render time rather than leaving them to review: + +1. **Resource requests and limits** — `oan-common.resources` fails the render + when `.Values.resources` is empty. This applies to data stores too, where the + requests land on the operator-managed pods. +2. **Liveness and readiness probes** — enabled by default in `oan-template`. + `oan-common.probeSpec` fails the render when an enabled probe declares no + handler, or declares more than one (which the API server would otherwise + reject at apply time, long after the render looked fine). +3. **A ServiceAccount per service** — never the namespace `default`. Attach IRSA + role ARNs via `serviceAccount.annotations`. +4. **Standard labels** — from `oan-common.labels`, giving every resource + `app.kubernetes.io/*` plus `app.kubernetes.io/part-of: oan`. + +## Secrets + +No secret value is ever committed to this repository — not in `values.yaml`, not +in a per-environment values file. + +- **No chart in this repository renders a Secret.** Charts only *reference* + Secrets by name, so the question of how secret material gets into the cluster + is answered once, outside the charts, rather than differently per chart. +- `envFromSecrets` lists names of Secrets whose keys become environment + variables. `secretEnv` maps one Secret key to one variable name, for when the + producing key and the expected variable differ. +- `envConfig` is for non-secret configuration only. It lands in a ConfigMap. + +## Validation + +`./scripts/lint-charts.sh` runs `helm lint --strict` on every chart and +`helm template` on every application chart. CI runs the same script on pull +requests and on pushes to `main` and `development`, so a chart that fails +locally fails the same way in CI. + +Because service charts depend on `oan-common` through +`file://../oan-common`, and the packaged dependency is not committed, an edit to +`oan-common` is only visible to a consuming chart after +`helm dependency update charts/` (or a run of the lint script). + +## Traceability + +Chart work follows the repository's git conventions: branch +`feat/-`, commits carrying `[#]`, PR body closing the issue. +The chart's changelog entry and the issue number are the two ends of the same +thread. diff --git a/scripts/lint-charts.sh b/scripts/lint-charts.sh new file mode 100755 index 0000000..dddf9a2 --- /dev/null +++ b/scripts/lint-charts.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Lint and render every chart under charts/. +# +# Run locally exactly as CI runs it: +# ./scripts/lint-charts.sh +# +# Charts depending on oan-common via file://../oan-common carry no committed +# dependency artifact, so the dependency is rebuilt here before linting. That +# also means a local edit to oan-common is only picked up after this runs (or +# after `helm dependency update `). +# +# A chart whose defaults deliberately fail the render - because it requires a +# database host or a secret reference that has no safe default - supplies the +# minimum needed to template in ci/lint-values.yaml. Every file matching +# ci/*-values.yaml is rendered as a separate case, so a chart can cover several +# configurations. +set -euo pipefail + +cd "$(dirname "$0")/.." + +failed=0 + +for chart_dir in charts/*/; do + chart_dir="${chart_dir%/}" + [[ -f "$chart_dir/Chart.yaml" ]] || continue + + name="$(basename "$chart_dir")" + chart_type="$(helm show chart "$chart_dir" 2>/dev/null | awk '/^type:/ {print $2}')" + chart_type="${chart_type:-application}" + + echo "==> $name (type: $chart_type)" + + if grep -q '^dependencies:' "$chart_dir/Chart.yaml"; then + helm dependency update "$chart_dir" >/dev/null + fi + + # Render once per ci/*-values.yaml, or once with plain defaults if there are none. + shopt -s nullglob + ci_values=("$chart_dir"/ci/*-values.yaml) + shopt -u nullglob + + # Pass the same values to lint. Without them, lint renders bare defaults and + # reports the chart's own "this value is required" failures as [INFO] Fail + # lines - noise that looks like a broken chart but is the guardrail working. + # Written this way for bash 3.2 (macOS), where "${empty_array[@]}" under + # `set -u` is an unbound-variable error rather than an empty expansion. + lint_args=() + if [[ ${#ci_values[@]} -gt 0 ]]; then + for values in "${ci_values[@]}"; do + lint_args+=(-f "$values") + done + fi + + if ! helm lint --strict "$chart_dir" "${lint_args[@]+"${lint_args[@]}"}"; then + echo "!!! helm lint failed for $name" + failed=1 + continue + fi + + # Library charts render no resources of their own, so there is nothing to template. + if [[ "$chart_type" == "library" ]]; then + continue + fi + + if [[ ${#ci_values[@]} -eq 0 ]]; then + if ! helm template "$name" "$chart_dir" >/dev/null; then + echo "!!! helm template failed for $name" + failed=1 + fi + else + for values in "${ci_values[@]}"; do + echo " render: $(basename "$values")" + if ! helm template "$name" "$chart_dir" -f "$values" >/dev/null; then + echo "!!! helm template failed for $name with $(basename "$values")" + failed=1 + fi + done + fi +done + +if [[ "$failed" -ne 0 ]]; then + echo "chart validation FAILED" + exit 1 +fi + +echo "all charts linted and rendered successfully" From 8212a5c1790fbf153f04d4629b32c44db3e919eb Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:34:15 +0530 Subject: [PATCH 50/81] feat: add oan-common library chart and oan-template scaffold [#53] oan-common holds the shared helpers every OAN chart renders through - names, labels, image references, resources, and security contexts - so the service charts stay declarative. Resources are deliberately required rather than defaulted: an unset value fails the render instead of shipping an unbounded pod. oan-template is the copy-from starting point for a new service chart. --- charts/oan-common/.helmignore | 10 + charts/oan-common/CHANGELOG.md | 53 +++ charts/oan-common/Chart.yaml | 17 + charts/oan-common/README.md | 117 +++++ charts/oan-common/templates/_helpers.tpl | 402 ++++++++++++++++++ charts/oan-common/values.yaml | 151 +++++++ charts/oan-template/.helmignore | 10 + charts/oan-template/CHANGELOG.md | 37 ++ charts/oan-template/Chart.yaml | 22 + charts/oan-template/README.md | 121 ++++++ charts/oan-template/ci/lint-values.yaml | 9 + charts/oan-template/templates/NOTES.txt | 28 ++ charts/oan-template/templates/_helpers.tpl | 45 ++ charts/oan-template/templates/configmap.yaml | 12 + charts/oan-template/templates/deployment.yaml | 79 ++++ charts/oan-template/templates/ingress.yaml | 41 ++ charts/oan-template/templates/service.yaml | 19 + .../templates/serviceaccount.yaml | 13 + charts/oan-template/values.yaml | 155 +++++++ 19 files changed, 1341 insertions(+) create mode 100644 charts/oan-common/.helmignore create mode 100644 charts/oan-common/CHANGELOG.md create mode 100644 charts/oan-common/Chart.yaml create mode 100644 charts/oan-common/README.md create mode 100644 charts/oan-common/templates/_helpers.tpl create mode 100644 charts/oan-common/values.yaml create mode 100644 charts/oan-template/.helmignore create mode 100644 charts/oan-template/CHANGELOG.md create mode 100644 charts/oan-template/Chart.yaml create mode 100644 charts/oan-template/README.md create mode 100644 charts/oan-template/ci/lint-values.yaml create mode 100644 charts/oan-template/templates/NOTES.txt create mode 100644 charts/oan-template/templates/_helpers.tpl create mode 100644 charts/oan-template/templates/configmap.yaml create mode 100644 charts/oan-template/templates/deployment.yaml create mode 100644 charts/oan-template/templates/ingress.yaml create mode 100644 charts/oan-template/templates/service.yaml create mode 100644 charts/oan-template/templates/serviceaccount.yaml create mode 100644 charts/oan-template/values.yaml diff --git a/charts/oan-common/.helmignore b/charts/oan-common/.helmignore new file mode 100644 index 0000000..7e6c23d --- /dev/null +++ b/charts/oan-common/.helmignore @@ -0,0 +1,10 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig diff --git a/charts/oan-common/CHANGELOG.md b/charts/oan-common/CHANGELOG.md new file mode 100644 index 0000000..50a83af --- /dev/null +++ b/charts/oan-common/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +All notable changes to the `oan-common` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Added +- `oan-common.image` now fails the render when `image.repository` is empty. + Previously an empty repository produced a syntactically valid but meaningless + reference such as `ghcr.io/:v2.0.0`, which Helm and the API server both accept + and which only surfaces later as an `ImagePullBackOff` - long after the deploy + appeared to succeed. + +### Removed +- `oan-common.externalsecret`, `oan-common.externalSecret.enabled` and + `oan-common.externalSecret.targetName`, along with the `externalSecrets` value + block. External Secrets Operator is not installed in any OAN cluster, so no + consuming chart could exercise them. + + The helpers that reference existing Secrets - `oan-common.env` for `secretEnv`, + and each chart's own `*Secret.name` settings - are unchanged. + +## [0.1.0] - 2026-08-31 + +### Added +- Initial release of the `oan-common` library chart. +- Name, fullname, chart, standard label, selector label, annotation, and + namespace helpers. +- Image reference and image pull secret helpers, with digest pinning + (`image.digest`) taking precedence over `image.tag`. +- Service account name/enabled helpers. +- Env ConfigMap name/data helpers and an `envConfig` checksum annotation helper. +- `oan-common.env`, rendering container env entries from `secretEnv` (mapping an + env var name to a specific Secret key) and `extraEnv` (raw passthrough), and + failing the render when a `secretEnv` entry is missing its name or key. +- `oan-common.resources`, which fails the render when `resources` is empty so no + component can ship without a resource contract. +- `oan-common.probes` and `oan-common.probeSpec`, passing every probe field + through verbatim and failing the render when an enabled probe declares no + handler or more than one. +- Pod-level and container-level security context helpers, gated on `enabled`. +- `oan-common.externalsecret`, rendering a complete External Secrets Operator + `ExternalSecret`, with target name and enabled helpers, and render-time + validation of `secretStoreRef.name` and `data`/`dataFrom`. +- `oan-common.waitFor`, rendering init containers that block startup until TCP + and HTTP dependencies are reachable - the missing equivalent of compose's + `depends_on: condition: service_healthy`. Checks are passed in explicitly so a + consuming chart derives host and URL from its own settings rather than + duplicating them. +- Deployment and Ingress apiVersion helpers. diff --git a/charts/oan-common/Chart.yaml b/charts/oan-common/Chart.yaml new file mode 100644 index 0000000..87c04ac --- /dev/null +++ b/charts/oan-common/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: oan-common +description: Common Helm library chart for OpenAgriNet (OAN) services +type: library +version: 0.2.0 +appVersion: "0.1.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - common + - library +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/oan-common/README.md b/charts/oan-common/README.md new file mode 100644 index 0000000..a254a00 --- /dev/null +++ b/charts/oan-common/README.md @@ -0,0 +1,117 @@ +# oan-common + +The common Helm **library chart** for OpenAgriNet (OAN) services. + +`oan-common` renders **no resources of its own** and is never installed +directly. Service charts declare it as a dependency and call its named template +helpers, so names, labels, image references, probes, resource contracts, and +secret wiring are identical across every OAN chart. + +## Using it in a chart + +1. Declare the dependency in your chart's `Chart.yaml`: + + ```yaml + dependencies: + - name: oan-common + version: "0.1.x" + repository: "file://../oan-common" + ``` + +2. Pull it in: + + ```bash + helm dependency update charts/ + ``` + + The packaged dependency is not committed, so re-run this after every edit to + `oan-common` — otherwise your chart keeps rendering against a stale copy. + +3. Define thin chart-local wrappers in your `templates/_helpers.tpl` that + delegate to the library: + + ```yaml + {{- define "my-service.fullname" -}} + {{- include "oan-common.fullname" . -}} + {{- end }} + ``` + + See [`oan-template`](../oan-template) for a complete, copy-ready example. + +## Helpers + +### Names and labels + +| Helper | Purpose | +|---|---| +| `oan-common.name` | Chart name, honoring `nameOverride` | +| `oan-common.fullname` | Fully qualified name (`-`), honoring `fullnameOverride` | +| `oan-common.chart` | `name-version` string for the `helm.sh/chart` label | +| `oan-common.labels` | Standard `app.kubernetes.io/*` labels, `part-of: oan`, plus `commonLabels` | +| `oan-common.selectorLabels` | Pod/Service selector labels (name + instance) | +| `oan-common.annotations` | Renders `commonAnnotations` | +| `oan-common.namespace` | Release namespace | + +### Workload + +| Helper | Purpose | +|---|---| +| `oan-common.image` | Full image ref from `image.registry`/`repository`/`tag`, falling back to `Chart.appVersion` then `latest`. `image.digest` pins by digest and wins over the tag. **Fails the render when `repository` is empty** | +| `oan-common.imagePullSecrets` | Renders the `imagePullSecrets` block from `image.pullSecrets` | +| `oan-common.resources` | Renders `resources`. **Fails the render when empty** — every OAN component must declare a resource contract | +| `oan-common.probes` | Renders every enabled probe block (startup, liveness, readiness) for a container spec | +| `oan-common.probeSpec` | Renders one probe. Takes `(dict "probe" "name" "chart" .Chart.Name)` | +| `oan-common.podSecurityContext` | Pod-level security context, only when `podSecurityContext.enabled` | +| `oan-common.securityContext` | Container-level security context, only when `securityContext.enabled` | + +Probes pass every field except `enabled` through verbatim, so any handler +(`httpGet`, `tcpSocket`, `exec`, `grpc`) and any timing field works. Two +render-time guardrails apply: + +- An enabled probe with **no** handler fails the render. +- An enabled probe with **more than one** handler fails the render. This is the + common trap: Helm merges maps, so overriding a default `httpGet` probe with + `tcpSocket` leaves both in the merged value and the API server rejects it at + apply time. Null out the default you are replacing: + + ```bash + --set livenessProbe.tcpSocket.port=http --set livenessProbe.httpGet=null + ``` + +### Service account + +| Helper | Purpose | +|---|---| +| `oan-common.serviceAccount.name` | Service account name (generated, overridden, or `default` when disabled) | +| `oan-common.serviceAccount.enabled` | Emits `true` when a ServiceAccount should be created | + +### Configuration and secrets + +| Helper | Purpose | +|---|---| +| `oan-common.env` | Container `env` entries from `secretEnv` (env var name -> secret key) and `extraEnv` (raw passthrough) | +| `oan-common.envConfigMapName` | Name of the env ConfigMap (`-env`) | +| `oan-common.envConfigMapData` | Renders `envConfig` into ConfigMap `data` entries | +| `oan-common.checksumAnnotation` | Checksum of `envConfig`, to roll pods when config changes | + +### apiVersions + +`oan-common.deployment.apiVersion` and `oan-common.ingress.apiVersion` keep those +in one place, so a Kubernetes upgrade is a single edit. + +## Value schema + +The helpers read the keys documented in [`values.yaml`](./values.yaml): +`nameOverride`, `fullnameOverride`, `image.*` (including `digest`), `serviceAccount.*`, `envConfig`, +`resources`, `secretEnv`, `extraEnv`, `livenessProbe`/`readinessProbe`/`startupProbe`, +`podSecurityContext`, `securityContext`, `waitFor`, `commonLabels`, and +`commonAnnotations`. A consuming chart inherits this schema and extends it with +its own keys (`replicaCount`, `service`, `ingress`, ...). + +## Versioning + +Consumers pin `version: "0.1.x"`. Ship helper additions as PATCH/MINOR; reserve +MAJOR for renaming or changing the behaviour of an existing helper, since that +forces every consuming chart to update its pin. Every change needs a `version` +bump in `Chart.yaml` and an entry in [`CHANGELOG.md`](./CHANGELOG.md) — see +[`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/oan-common/templates/_helpers.tpl b/charts/oan-common/templates/_helpers.tpl new file mode 100644 index 0000000..6504773 --- /dev/null +++ b/charts/oan-common/templates/_helpers.tpl @@ -0,0 +1,402 @@ +{{/* +# ============================================================================ +# OAN COMMON LIBRARY CHART - SHARED HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: reusable template helpers consumed by every OAN service chart. +# This chart renders no resources of its own. +# GitHub: https://github.com/OpenAgriNet/helmcharts/blob/main/charts/oan-common/templates/_helpers.tpl +# ============================================================================ +*/}} + +{{/* +Chart name, honoring nameOverride. +Usage: {{ include "oan-common.name" . }} +*/}} +{{- define "oan-common.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified app name (-), honoring fullnameOverride. +Truncated to 63 chars for the DNS label limit. +Usage: {{ include "oan-common.fullname" . }} +*/}} +{{- define "oan-common.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart name and version for the helm.sh/chart label. +Usage: {{ include "oan-common.chart" . }} +*/}} +{{- define "oan-common.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Standard labels for every resource. +Usage: {{ include "oan-common.labels" . | nindent 4 }} +*/}} +{{- define "oan-common.labels" -}} +helm.sh/chart: {{ include "oan-common.chart" . }} +{{ include "oan-common.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: oan +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels for pods and services. These are immutable on a Deployment +selector, so nothing environment-specific belongs here. +Usage: {{ include "oan-common.selectorLabels" . | nindent 4 }} +*/}} +{{- define "oan-common.selectorLabels" -}} +app.kubernetes.io/name: {{ include "oan-common.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common annotations for every resource. +Usage: {{ include "oan-common.annotations" . | nindent 4 }} +*/}} +{{- define "oan-common.annotations" -}} +{{- with .Values.commonAnnotations }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Name of the service account to use. +Usage: {{ include "oan-common.serviceAccount.name" . }} +*/}} +{{- define "oan-common.serviceAccount.name" -}} +{{- if .Values.serviceAccount.enabled }} +{{- default (include "oan-common.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Emits "true" when a ServiceAccount should be created. +Usage: {{ if include "oan-common.serviceAccount.enabled" . }} +*/}} +{{- define "oan-common.serviceAccount.enabled" -}} +{{- if .Values.serviceAccount.enabled }} +{{- true }} +{{- end }} +{{- end }} + +{{/* +Full image reference: /:, or +/@ when image.digest is set. + +A digest wins over a tag, so an environment can pin an exact image without +having to blank out the tag. + +Tag falls back to Chart.appVersion, then "latest". +Usage: {{ include "oan-common.image" . }} +*/}} +{{- define "oan-common.image" -}} +{{- $registry := .Values.image.registry | default "" }} +{{- $repository := .Values.image.repository | default "" }} +{{- if not $repository }} +{{/* +Without this, an empty repository renders a syntactically valid but meaningless +reference - "ghcr.io/:v2.0.0" - which Helm and the API server both accept. The +failure only surfaces later as an ImagePullBackOff, long after the deploy looked +successful. Fail here instead. +*/}} +{{- fail (printf "%s: image.repository is required - set image.registry/repository/tag for this environment" .Chart.Name) }} +{{- end }} +{{- if $registry }} +{{- $repository = printf "%s/%s" $registry $repository }} +{{- end }} +{{- with .Values.image.digest }} +{{- printf "%s@%s" $repository . }} +{{- else }} +{{- printf "%s:%s" $repository (.Values.image.tag | default $.Chart.AppVersion | default "latest") }} +{{- end }} +{{- end }} + +{{/* +imagePullSecrets block, rendered only when image.pullSecrets is non-empty. +Usage: {{ include "oan-common.imagePullSecrets" . | nindent 6 }} +*/}} +{{- define "oan-common.imagePullSecrets" -}} +{{- with .Values.image.pullSecrets }} +imagePullSecrets: +{{- range . }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Name of the env ConfigMap (-env). +Usage: {{ include "oan-common.envConfigMapName" . }} +*/}} +{{- define "oan-common.envConfigMapName" -}} +{{- printf "%s-env" (include "oan-common.fullname" .) }} +{{- end }} + +{{/* +envConfig rendered as ConfigMap data entries. +Usage: {{ include "oan-common.envConfigMapData" . | nindent 2 }} +*/}} +{{- define "oan-common.envConfigMapData" -}} +{{- range $key, $value := .Values.envConfig }} +{{ $key }}: {{ $value | quote }} +{{- end }} +{{- end }} + +{{/* +Checksum of envConfig, so a config change rolls the pods. +Usage: checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} +*/}} +{{- define "oan-common.checksumAnnotation" -}} +{{- $envConfig := .Values.envConfig | default dict }} +{{- $envConfig | toJson | sha256sum }} +{{- end }} + +{{/* +Container env entries built from `secretEnv` and `extraEnv`. + +`secretEnv` maps an environment variable name to a secret key, for the common +case where the producing secret's key name differs from the variable the app +expects (CNPG writes `password`; Sunbird RC wants `connectionInfo_password`): + + secretEnv: + connectionInfo_password: + name: registry-db-app + key: password + optional: false # optional + +`extraEnv` is a raw list of env entries, passed through verbatim for anything +this schema does not cover (fieldRef, resourceFieldRef, plain values). + +Usage: + {{- with (include "oan-common.env" . | trim) }} + env: + {{- . | nindent 12 }} + {{- end }} +*/}} +{{- define "oan-common.env" -}} +{{- range $name, $ref := .Values.secretEnv }} +{{- if not $ref.name }} +{{- fail (printf "secretEnv.%s.name is required - it must name the Secret holding the value" $name) }} +{{- end }} +{{- if not $ref.key }} +{{- fail (printf "secretEnv.%s.key is required - it must name the key inside Secret %q" $name $ref.name) }} +{{- end }} +- name: {{ $name }} + valueFrom: + secretKeyRef: + name: {{ $ref.name }} + key: {{ $ref.key }} + {{- if hasKey $ref "optional" }} + optional: {{ $ref.optional }} + {{- end }} +{{- end }} +{{- with .Values.extraEnv }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Init containers that block startup until dependencies are reachable. + +Kubernetes has no equivalent of compose's `depends_on: condition: +service_healthy`. Without this a pod starts before its database or its identity +provider is up, fails, and crashloops with backoff - which recovers on its own +but makes a first install look broken and slows every restart. + +Takes the checks explicitly, so a consuming chart derives them from its own +settings (database.host, keycloak.url) instead of making the operator retype +values that would then be free to drift: + + {{- include "oan-common.waitFor" (dict "ctx" . "tcp" $tcp "http" $http) }} + +where $tcp entries are {name, host, port} and $http entries are {name, url}. + +TCP checks use `nc -z`; HTTP checks use `wget --spider`. Both loop until success +or waitFor.timeoutSeconds, then fail the pod so the reason is visible in +`kubectl describe` rather than buried in a crashloop. +*/}} +{{- define "oan-common.waitFor" -}} +{{- $ctx := .ctx -}} +{{- $w := $ctx.Values.waitFor | default dict -}} +{{- if $w.enabled -}} +{{- $img := $w.image | default dict -}} +{{- $image := printf "%s/%s:%s" ($img.registry | default "docker.io") ($img.repository | default "busybox") ($img.tag | default "latest") -}} +{{- $timeout := $w.timeoutSeconds | default 300 -}} +{{- $interval := $w.intervalSeconds | default 3 -}} +{{- $res := $w.resources | default dict -}} +{{- range .tcp }} +{{- if not .host }} +{{- fail (printf "waitFor: the %q check has no host. Set the setting it derives from (e.g. database.host), or disable the check." (.name | default "")) }} +{{- end }} +- name: {{ printf "wait-%s" (.name | default "tcp") | trunc 63 | trimSuffix "-" }} + image: {{ $image | quote }} + imagePullPolicy: {{ $img.pullPolicy | default "IfNotPresent" }} + command: + - /bin/sh + - -c + - | + deadline=$(( $(date +%s) + {{ $timeout }} )) + until nc -z {{ .host | quote }} {{ .port }}; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "timed out after {{ $timeout }}s waiting for {{ .name }} at {{ .host }}:{{ .port }}" + exit 1 + fi + echo "waiting for {{ .name }} at {{ .host }}:{{ .port }}" + sleep {{ $interval }} + done + echo "{{ .name }} is reachable" + resources: + {{- toYaml $res | nindent 4 }} +{{- end }} +{{- range .http }} +{{- if not .url }} +{{- fail (printf "waitFor: the %q check has no url. Set the setting it derives from (e.g. keycloak.url), or disable the check." (.name | default "")) }} +{{- end }} +- name: {{ printf "wait-%s" (.name | default "http") | trunc 63 | trimSuffix "-" }} + image: {{ $image | quote }} + imagePullPolicy: {{ $img.pullPolicy | default "IfNotPresent" }} + command: + - /bin/sh + - -c + - | + deadline=$(( $(date +%s) + {{ $timeout }} )) + until wget -q --spider --timeout=5 {{ .url | quote }}; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "timed out after {{ $timeout }}s waiting for {{ .name }} at {{ .url }}" + exit 1 + fi + echo "waiting for {{ .name }} at {{ .url }}" + sleep {{ $interval }} + done + echo "{{ .name }} is reachable" + resources: + {{- toYaml $res | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Resources block. +Fails the render when resources is empty: every OAN component must declare a +resource contract so the scheduler and the cluster autoscaler have real numbers +to work with. +Usage: {{ include "oan-common.resources" . | nindent 10 }} +*/}} +{{- define "oan-common.resources" -}} +{{- if not .Values.resources -}} +{{- fail (printf "%s: .Values.resources is required - every OAN component must declare requests and limits (see CONVENTIONS.md)" .Chart.Name) -}} +{{- end -}} +{{- toYaml .Values.resources }} +{{- end }} + +{{/* +Render one probe. Everything except the `enabled` flag is passed through +verbatim, so any handler (httpGet, tcpSocket, exec, grpc) and any timing field +works. + +Guardrail: Helm merges maps, so overriding a chart's default httpGet probe with +tcpSocket would leave BOTH handlers in the merged value - which the API server +rejects at apply time, long after the render looked fine. This fails the render +instead and tells you to null out the default. + +Usage: {{ include "oan-common.probeSpec" + (dict "probe" .Values.livenessProbe "name" "livenessProbe" "chart" .Chart.Name) | nindent 2 }} +*/}} +{{- define "oan-common.probeSpec" -}} +{{- $probe := .probe -}} +{{- $name := .name | default "probe" -}} +{{- $chart := .chart | default "chart" -}} +{{- $handlers := list -}} +{{- range $h := list "httpGet" "tcpSocket" "exec" "grpc" -}} +{{- if index $probe $h -}} +{{- $handlers = append $handlers $h -}} +{{- end -}} +{{- end -}} +{{- if gt (len $handlers) 1 -}} +{{- fail (printf "%s: %s declares %d handlers (%s) but a probe may declare only one. Null out the one you do not want, e.g. --set %s.httpGet=null" $chart $name (len $handlers) (join ", " $handlers) $name) -}} +{{- end -}} +{{- if eq (len $handlers) 0 -}} +{{- fail (printf "%s: %s is enabled but declares no handler. Set one of httpGet, tcpSocket, exec or grpc." $chart $name) -}} +{{- end -}} +{{- toYaml (omit $probe "enabled") }} +{{- end }} + +{{/* +All enabled probe blocks (startup, liveness, readiness) for a container spec. +Usage: {{ include "oan-common.probes" . | nindent 8 }} +*/}} +{{- define "oan-common.probes" -}} +{{- if and .Values.startupProbe .Values.startupProbe.enabled }} +startupProbe: + {{- include "oan-common.probeSpec" (dict "probe" .Values.startupProbe "name" "startupProbe" "chart" .Chart.Name) | nindent 2 }} +{{- end }} +{{- if and .Values.livenessProbe .Values.livenessProbe.enabled }} +livenessProbe: + {{- include "oan-common.probeSpec" (dict "probe" .Values.livenessProbe "name" "livenessProbe" "chart" .Chart.Name) | nindent 2 }} +{{- end }} +{{- if and .Values.readinessProbe .Values.readinessProbe.enabled }} +readinessProbe: + {{- include "oan-common.probeSpec" (dict "probe" .Values.readinessProbe "name" "readinessProbe" "chart" .Chart.Name) | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Pod-level security context, rendered only when enabled. +Usage: {{ include "oan-common.podSecurityContext" . | nindent 8 }} +*/}} +{{- define "oan-common.podSecurityContext" -}} +{{- if and .Values.podSecurityContext .Values.podSecurityContext.enabled }} +{{- toYaml (omit .Values.podSecurityContext "enabled") }} +{{- end }} +{{- end }} + +{{/* +Container-level security context, rendered only when enabled. +Usage: {{ include "oan-common.securityContext" . | nindent 10 }} +*/}} +{{- define "oan-common.securityContext" -}} +{{- if and .Values.securityContext .Values.securityContext.enabled }} +{{- toYaml (omit .Values.securityContext "enabled") }} +{{- end }} +{{- end }} + +{{/* +Release namespace. +Usage: {{ include "oan-common.namespace" . }} +*/}} +{{- define "oan-common.namespace" -}} +{{- .Release.Namespace }} +{{- end }} + +{{/* +apiVersion helpers, so a bump lands in one place. +*/}} +{{- define "oan-common.deployment.apiVersion" -}} +apps/v1 +{{- end }} + +{{- define "oan-common.ingress.apiVersion" -}} +networking.k8s.io/v1 +{{- end }} diff --git a/charts/oan-common/values.yaml b/charts/oan-common/values.yaml new file mode 100644 index 0000000..f1517ee --- /dev/null +++ b/charts/oan-common/values.yaml @@ -0,0 +1,151 @@ +# ============================================================================ +# Default values for the oan-common library chart. +# +# A library chart renders no resources of its own. These keys define the value +# schema that consuming charts inherit, and are what the helpers in +# templates/_helpers.tpl read. Consuming charts may extend this schema with +# their own keys (service, ingress, replicaCount, ...). +# ============================================================================ + +# Override the chart name used in names and labels +nameOverride: "" +# Override the generated fullname (default: -) +fullnameOverride: "" + +# Image configuration. Consumed by `oan-common.image`. +image: + # Container registry, e.g. "ghcr.io" or ".dkr.ecr.ap-south-1.amazonaws.com" + registry: "" + # Image repository, e.g. "openagrinet/registry-service" + repository: "" + # Image tag. Falls back to Chart.appVersion, then "latest". + tag: "" + # Image digest (sha256:...). Takes precedence over tag when set, so an + # environment can pin an exact image without blanking the tag. + digest: "" + pullPolicy: IfNotPresent + # Names of image pull secrets in the release namespace + pullSecrets: [] + +# Service account configuration. Consumed by `oan-common.serviceAccount.*`. +serviceAccount: + # Create a dedicated ServiceAccount for this release + enabled: true + # Explicit name. Defaults to the chart fullname when empty. + name: "" + # Annotations, e.g. eks.amazonaws.com/role-arn for IRSA + annotations: {} + automountServiceAccountToken: true + +# Non-secret environment configuration. Rendered into a ConfigMap and exposed +# to the container with envFrom. NEVER put secrets here. +envConfig: {} + +# Resource requests and limits. +# REQUIRED on every OAN component: `oan-common.resources` fails the render when +# this is empty, so a chart cannot ship without a resource contract. +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# Probes. REQUIRED on every OAN component that serves traffic. +# Every field other than `enabled` is passed through to the probe verbatim, so +# any handler (httpGet, tcpSocket, exec, grpc) and any timing field is allowed. +livenessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +startupProbe: + enabled: false + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + +# Pod-level security context. Every field other than `enabled` is passed +# through verbatim. +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +# Container-level security context. Every field other than `enabled` is passed +# through verbatim. +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + +# Init containers that block startup until dependencies are reachable, standing +# in for compose's `depends_on: condition: service_healthy`, which Kubernetes has +# no equivalent of. Consumed by `oan-common.waitFor`. +waitFor: + enabled: false + image: + registry: docker.io + # Needs `nc`, `wget` and `date` - busybox has all three. + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + # Give up after this long, so a genuinely missing dependency surfaces as a + # failed init container rather than an endless wait. + timeoutSeconds: 300 + intervalSeconds: 3 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Which dependencies to wait for is decided by the consuming chart, which + # derives host and URL from its own settings. See that chart's values. + +# Individual environment variables sourced from Secret keys, for when the +# secret's key name differs from the variable the application expects +# (CNPG writes `password`; Sunbird RC wants `connectionInfo_password`). +# Consumed by `oan-common.env`. +secretEnv: {} + # connectionInfo_password: + # name: registry-db-app + # key: password + +# Raw env entries, passed through verbatim (fieldRef, plain values, ...). +extraEnv: [] + +# Extra labels added to every resource +commonLabels: {} +# Extra annotations added to every resource +commonAnnotations: {} diff --git a/charts/oan-template/.helmignore b/charts/oan-template/.helmignore new file mode 100644 index 0000000..7e6c23d --- /dev/null +++ b/charts/oan-template/.helmignore @@ -0,0 +1,10 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig diff --git a/charts/oan-template/CHANGELOG.md b/charts/oan-template/CHANGELOG.md new file mode 100644 index 0000000..2dfa74c --- /dev/null +++ b/charts/oan-template/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +All notable changes to the `oan-template` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +### Added +- Initial release of the `oan-template` reference/starter application chart. +- Depends on the `oan-common` library chart via `file://../oan-common`. +- Templates for Deployment, Service, ServiceAccount, env ConfigMap, optional + ESO ExternalSecret, and optional Ingress, all wired through `oan-common` + helpers. +- Liveness and readiness probes and resource requests/limits enabled by default, + per the deployment epic's requirement for every component. +- Pod-level and container-level security contexts, off by default and enabled + per environment. +- Scheduling controls (`nodeSelector`, `tolerations`, `affinity`) and extra + `podLabels` / `podAnnotations`. +- `envFromSecrets` for pre-existing Secrets not managed by ESO, plus `secretEnv` + and `extraEnv` for individual environment variables. +- Fully commented `values.yaml` and a `NOTES.txt` summarising what was rendered. diff --git a/charts/oan-template/Chart.yaml b/charts/oan-template/Chart.yaml new file mode 100644 index 0000000..6ec645e --- /dev/null +++ b/charts/oan-template/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: oan-template +description: Reference starter chart for OpenAgriNet (OAN) services, built on oan-common +type: application +version: 0.2.0 +appVersion: "1.0.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - template + - starter + - reference +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/oan-template/README.md b/charts/oan-template/README.md new file mode 100644 index 0000000..c85a25d --- /dev/null +++ b/charts/oan-template/README.md @@ -0,0 +1,121 @@ +# oan-template + +The **reference / starter** Helm chart for OpenAgriNet (OAN) services. + +`oan-template` is a complete, working example of an OAN service chart built on +the [`oan-common`](../oan-common) library chart. It is meant to be **copied and +adapted** — not installed as-is. + +## What it renders + +| Resource | File | Notes | +|---|---|---| +| Deployment | `templates/deployment.yaml` | Probes and resources always present; security contexts gated on `enabled` | +| Service | `templates/service.yaml` | `ClusterIP` by default | +| ServiceAccount | `templates/serviceaccount.yaml` | Created when `serviceAccount.enabled` | +| ConfigMap | `templates/configmap.yaml` | Built from `envConfig`; checksum rolls pods on change | +| Ingress | `templates/ingress.yaml` | Optional, off by default | + +Names, labels, image refs, probes, resources, and secret wiring all come from +`oan-common` helpers, so every chart derived from this template stays consistent. + +## How it depends on oan-common + +`Chart.yaml` declares: + +```yaml +dependencies: + - name: oan-common + version: "0.1.x" + repository: "file://../oan-common" +``` + +The chart-local helpers in `templates/_helpers.tpl` are thin wrappers that +delegate to the library (`oan-template.fullname` → `oan-common.fullname`). + +## Create your own chart from this template + +1. Copy the directory and rename it: + + ```bash + cp -r charts/oan-template charts/oan-my-service + ``` + +2. In `charts/oan-my-service/Chart.yaml`, set `name: oan-my-service` and + `appVersion` to the image tag you deploy by default. Keep the `oan-common` + dependency. + +3. Rename the chart-local helpers. Change only the **left side** of each + `define` in `templates/_helpers.tpl` — the `oan-common.*` include inside the + body stays, since that is the shared library you delegate to. Then update the + matching `include "oan-template..."` calls in the template YAML. Both at once, + from the repo root: + + ```bash + grep -rl 'oan-template\.' charts/oan-my-service | xargs sed -i '' 's/oan-template\./oan-my-service./g' + ``` + + (`sed -i ''` is the macOS form; on Linux use `sed -i`.) + +4. Set the real values in `values.yaml`: + - `image.registry` / `image.repository` / `image.tag` + - `service.port` / `service.targetPort` — your container's listen port + - `livenessProbe.httpGet.path` / `readinessProbe.httpGet.path` — your health endpoint + - `resources` — right-sized for the workload + - `envConfig` — non-secret configuration only + +5. Validate: + + ```bash + ./scripts/lint-charts.sh + helm template oan-my-service charts/oan-my-service + ``` + +## Try the template directly + +```bash +helm dependency update charts/oan-template +helm template demo charts/oan-template \ + --set image.repository=nginx --set image.tag=1.27 +``` + +## Configuration + +See [`values.yaml`](./values.yaml) for the full, commented schema. + +Two behaviours worth knowing before you override: + +- **`resources` is mandatory.** Emptying it fails the render by design. +- **Switching a probe handler needs the default nulled out.** Helm merges maps, + so replacing the default `httpGet` probe with `tcpSocket` leaves both handlers + in the merged value. The render fails with an explanatory error; null the one + you are replacing: + + ```bash + --set livenessProbe.tcpSocket.port=http --set livenessProbe.httpGet=null + ``` + +## Secrets + +Never put secret values in `envConfig` or in any committed values file. This +chart renders no Secrets at all — it only *references* them, so they are created +by whatever manages secrets in that environment. + +Two ways to consume one: + +```yaml +# whole Secret injected as env vars +envFromSecrets: + - my-service-db + +# a single key mapped to a specific variable name, for when they differ +secretEnv: + DB_PASSWORD: + name: my-service-db + key: password +``` + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/oan-template/ci/lint-values.yaml b/charts/oan-template/ci/lint-values.yaml new file mode 100644 index 0000000..17cad2c --- /dev/null +++ b/charts/oan-template/ci/lint-values.yaml @@ -0,0 +1,9 @@ +# Minimum values that let `helm template` run in CI. +# +# This chart's image.repository is intentionally empty - it is a starter chart, +# and a chart with no image is not a deployable thing. The render fails without +# one, so CI supplies a placeholder. +image: + registry: docker.io + repository: nginx + tag: "1.27" diff --git a/charts/oan-template/templates/NOTES.txt b/charts/oan-template/templates/NOTES.txt new file mode 100644 index 0000000..1e8e8be --- /dev/null +++ b/charts/oan-template/templates/NOTES.txt @@ -0,0 +1,28 @@ +Thank you for installing {{ .Chart.Name }} (release: {{ .Release.Name }}). + +Rendered from the oan-template reference chart. + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "oan-template.fullname" . }} ({{ .Values.replicaCount }} replica(s)) + - Service/{{ include "oan-template.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "oan-template.envConfigMapName" . }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "oan-template.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "oan-template.fullname" . }} +{{- end }} + +Image in use: + {{ include "oan-template.image" . }} +{{- if not .Values.image.repository }} + +WARNING: image.repository is empty. Set image.registry/repository/tag before a +real deployment, e.g. --set image.repository=openagrinet/my-service +{{- end }} + +Check the rollout: + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "oan-template.fullname" . }} + +Reach the service in-cluster: + {{ include "oan-template.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} diff --git a/charts/oan-template/templates/_helpers.tpl b/charts/oan-template/templates/_helpers.tpl new file mode 100644 index 0000000..29ec803 --- /dev/null +++ b/charts/oan-template/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* +# ============================================================================ +# OAN TEMPLATE CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers that delegate to oan-common. +# +# When you copy this chart, rename every "oan-template." define below to your +# service name and update the matching include calls in the template YAML. +# Change only the LEFT side of each define - the oan-common include inside the +# body is the shared library you delegate to. +# ============================================================================ +*/}} + +{{- define "oan-template.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "oan-template.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "oan-template.chart" -}} +{{- include "oan-common.chart" . -}} +{{- end }} + +{{- define "oan-template.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "oan-template.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "oan-template.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "oan-template.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "oan-template.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + diff --git a/charts/oan-template/templates/configmap.yaml b/charts/oan-template/templates/configmap.yaml new file mode 100644 index 0000000..656ef85 --- /dev/null +++ b/charts/oan-template/templates/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "oan-template.envConfigMapName" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "oan-common.envConfigMapData" . | trim | nindent 2 }} diff --git a/charts/oan-template/templates/deployment.yaml b/charts/oan-template/templates/deployment.yaml new file mode 100644 index 0000000..5de7fa3 --- /dev/null +++ b/charts/oan-template/templates/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "oan-template.fullname" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "oan-template.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "oan-template.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "oan-template.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "oan-template.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + {{- with (include "oan-common.env" . | trim) }} + env: + {{- . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ include "oan-template.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/oan-template/templates/ingress.yaml b/charts/oan-template/templates/ingress.yaml new file mode 100644 index 0000000..964e961 --- /dev/null +++ b/charts/oan-template/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "oan-template.fullname" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "oan-template.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/oan-template/templates/service.yaml b/charts/oan-template/templates/service.yaml new file mode 100644 index 0000000..109557c --- /dev/null +++ b/charts/oan-template/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "oan-template.fullname" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "oan-template.selectorLabels" . | nindent 4 }} diff --git a/charts/oan-template/templates/serviceaccount.yaml b/charts/oan-template/templates/serviceaccount.yaml new file mode 100644 index 0000000..ca5974c --- /dev/null +++ b/charts/oan-template/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "oan-template.serviceAccountName" . }} + labels: + {{- include "oan-template.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/oan-template/values.yaml b/charts/oan-template/values.yaml new file mode 100644 index 0000000..8a06373 --- /dev/null +++ b/charts/oan-template/values.yaml @@ -0,0 +1,155 @@ +# ============================================================================ +# oan-template - default values +# +# This is a REFERENCE / STARTER chart. Copy the whole `oan-template` directory, +# rename it to your service, then adjust the values below. It renders a standard +# stateless service: Deployment + Service + ServiceAccount + env ConfigMap + +# optional Ingress, all wired through oan-common. +# +# Minimum you must change for a real deployment: +# - image.registry / image.repository / image.tag +# - service.port / service.targetPort (your container's listen port) +# - livenessProbe / readinessProbe httpGet.path (your health endpoint) +# - resources (right-size for the workload) +# - envConfig (non-secret configuration only) +# ============================================================================ + +replicaCount: 1 + +# Image configuration. Override registry/repository/tag per environment. +image: + # e.g. "ghcr.io" or ".dkr.ecr.ap-south-1.amazonaws.com" + registry: "" + # e.g. "openagrinet/my-service" + repository: "" + # Leave empty to fall back to Chart.appVersion + tag: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + # For IRSA on EKS: + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + annotations: {} + automountServiceAccountToken: true + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + annotations: {} + +# Ingress. Disabled by default; enable per environment. +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: my-service.local + paths: + - path: / + pathType: Prefix + tls: [] + +# Resource requests and limits. REQUIRED - the render fails if this is empty. +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# Probes. Point httpGet.path at your service's health endpoint. +# Any handler (httpGet, tcpSocket, exec, grpc) and any timing field is passed +# through verbatim; only `enabled` is stripped. +livenessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +readinessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# Enable for slow-starting services so liveness does not kill them mid-boot. +startupProbe: + enabled: false + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + +# Pod-level security context. Off by default - enable once the image is +# verified to run as a non-root user. +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +# Container-level security context. Off by default; turn on +# readOnlyRootFilesystem once the app writes only to mounted volumes. +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + +# Scheduling +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# ============================================================================ +# ENVIRONMENT CONFIGURATION +# Rendered into a ConfigMap and exposed to the container via envFrom. Pods +# restart automatically when these change (checksum annotation). +# +# NEVER put secrets here. Reference Secrets via envFromSecrets or secretEnv. +# ============================================================================ +envConfig: + LOG_LEVEL: "info" + SERVICE_NAME: "oan-template" + +# Names of pre-existing Secrets in this namespace to inject via envFrom, for +# secrets not managed by ESO. +envFromSecrets: [] + +# Individual env vars sourced from specific Secret keys. Use this when the +# secret's key name differs from the env var the application expects. +secretEnv: {} + # DB_PASSWORD: + # name: my-service-db + # key: password + +# Raw env entries, passed through verbatim. +extraEnv: [] From 863386e11fe417ea2793cb6e2bb23cfe4675ede2 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:34:35 +0530 Subject: [PATCH 51/81] feat: add postgresql-cnpg and postgresql-migration charts [#53] postgresql-cnpg renders one CloudNativePG Cluster per release, with optional ScheduledBackup and Barman Cloud ObjectStore. Neither chart renders a password: every credential is a reference to a Secret, and the render fails when one is unset rather than defaulting. Applications connect as the owner of their own database, never as postgres. Extensions that require superuser are created once through bootstrap.postInitApplicationSQL, which the operator runs during bootstrap, so no long-lived role holds the privilege. postgresql-migration runs Flyway as a Job across the databases the cluster chart created. Both target directories carry no SQL yet - Sunbird RC and Keycloak each manage their own schema - so the Job currently skips them and the targets exist as a versioned home. --- charts/postgresql-cnpg/.helmignore | 13 + charts/postgresql-cnpg/CHANGELOG.md | 125 ++++++++ charts/postgresql-cnpg/Chart.yaml | 31 ++ charts/postgresql-cnpg/README.md | 220 ++++++++++++++ charts/postgresql-cnpg/ci/lint-values.yaml | 10 + .../examples/discovery-db.dev.yaml | 139 +++++++++ .../examples/registry-db.dev.yaml | 100 +++++++ charts/postgresql-cnpg/templates/NOTES.txt | 41 +++ charts/postgresql-cnpg/templates/_helpers.tpl | 110 +++++++ charts/postgresql-cnpg/templates/cluster.yaml | 202 +++++++++++++ .../postgresql-cnpg/templates/database.yaml | 64 +++++ .../templates/objectstore.yaml | 41 +++ .../templates/scheduledbackup.yaml | 27 ++ charts/postgresql-cnpg/values.yaml | 271 ++++++++++++++++++ charts/postgresql-migration/.helmignore | 13 + charts/postgresql-migration/CHANGELOG.md | 104 +++++++ charts/postgresql-migration/Chart.yaml | 27 ++ charts/postgresql-migration/README.md | 160 +++++++++++ .../postgresql-migration/ci/lint-values.yaml | 7 + .../ci/no-hook-values.yaml | 11 + .../examples/registry-stack.dev.yaml | 55 ++++ .../files/migrations/01-registry/README.md | 13 + .../files/migrations/02-keycloak/README.md | 10 + .../postgresql-migration/templates/NOTES.txt | 41 +++ .../templates/_helpers.tpl | 146 ++++++++++ .../templates/configmap-migrations.yaml | 29 ++ .../templates/configmap-script.yaml | 87 ++++++ .../templates/configmap.yaml | 25 ++ .../postgresql-migration/templates/job.yaml | 105 +++++++ .../templates/serviceaccount.yaml | 17 ++ charts/postgresql-migration/values.yaml | 194 +++++++++++++ 31 files changed, 2438 insertions(+) create mode 100644 charts/postgresql-cnpg/.helmignore create mode 100644 charts/postgresql-cnpg/CHANGELOG.md create mode 100644 charts/postgresql-cnpg/Chart.yaml create mode 100644 charts/postgresql-cnpg/README.md create mode 100644 charts/postgresql-cnpg/ci/lint-values.yaml create mode 100644 charts/postgresql-cnpg/examples/discovery-db.dev.yaml create mode 100644 charts/postgresql-cnpg/examples/registry-db.dev.yaml create mode 100644 charts/postgresql-cnpg/templates/NOTES.txt create mode 100644 charts/postgresql-cnpg/templates/_helpers.tpl create mode 100644 charts/postgresql-cnpg/templates/cluster.yaml create mode 100644 charts/postgresql-cnpg/templates/database.yaml create mode 100644 charts/postgresql-cnpg/templates/objectstore.yaml create mode 100644 charts/postgresql-cnpg/templates/scheduledbackup.yaml create mode 100644 charts/postgresql-cnpg/values.yaml create mode 100644 charts/postgresql-migration/.helmignore create mode 100644 charts/postgresql-migration/CHANGELOG.md create mode 100644 charts/postgresql-migration/Chart.yaml create mode 100644 charts/postgresql-migration/README.md create mode 100644 charts/postgresql-migration/ci/lint-values.yaml create mode 100644 charts/postgresql-migration/ci/no-hook-values.yaml create mode 100644 charts/postgresql-migration/examples/registry-stack.dev.yaml create mode 100644 charts/postgresql-migration/files/migrations/01-registry/README.md create mode 100644 charts/postgresql-migration/files/migrations/02-keycloak/README.md create mode 100644 charts/postgresql-migration/templates/NOTES.txt create mode 100644 charts/postgresql-migration/templates/_helpers.tpl create mode 100644 charts/postgresql-migration/templates/configmap-migrations.yaml create mode 100644 charts/postgresql-migration/templates/configmap-script.yaml create mode 100644 charts/postgresql-migration/templates/configmap.yaml create mode 100644 charts/postgresql-migration/templates/job.yaml create mode 100644 charts/postgresql-migration/templates/serviceaccount.yaml create mode 100644 charts/postgresql-migration/values.yaml diff --git a/charts/postgresql-cnpg/.helmignore b/charts/postgresql-cnpg/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/postgresql-cnpg/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/postgresql-cnpg/CHANGELOG.md b/charts/postgresql-cnpg/CHANGELOG.md new file mode 100644 index 0000000..63ca611 --- /dev/null +++ b/charts/postgresql-cnpg/CHANGELOG.md @@ -0,0 +1,125 @@ +# Changelog + +All notable changes to the `postgresql-cnpg` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.1] - 2026-09-01 + +### Added +- `examples/discovery-db.dev.yaml` - the discovery service's database. It pins + `dhi.io/pgvector:0.8-pg16`, because discovery-service's first migration + creates the `vector` extension and no stock CloudNativePG operand image + carries pgvector. + + Two things in it are load-bearing rather than decorative, and both were + verified against the image rather than assumed: + + - `postInitApplicationSQL` creates `vector` and `pg_trgm`. `pg_trgm` is + marked `trusted`, so a database owner may create it; **`vector` is not**, + and `CREATE EXTENSION vector` as a non-superuser fails with "permission + denied to create extension". The service connects as the bootstrap owner + and `enableSuperuserAccess` is false, so without this the first statement + of its migration fails and the pod crashloops on what looks like a + credentials problem. CNPG runs this hook as superuser in the newly created + database, which is the one moment superuser is available without granting + it to anything long-lived. + - `postgresUID`/`postgresGID` are 70, the image's postgres user. CNPG + defaults to 26 and applies it as the pod's `runAsUser` and `fsGroup`. + + The image is a general-purpose PostgreSQL image, not a CNPG operand image. It + meets CNPG's documented requirements (initdb, postgres, pg_ctl, + pg_controldata, pg_basebackup and du on PATH; CNPG overrides its entrypoint) + but has not been run under the operator, and it ships no barman-cloud - which + rules out the in-core backup method, though not the Barman Cloud Plugin. + + No template changed, so this affects no existing release. + +## [0.2.0] - 2026-08-31 + +### Added +- `databases`, rendering a CNPG `Database` object per entry, so one cluster can + host several service databases. `bootstrap.initdb` creates only one; this covers + the rest. Requires CNPG >= 1.25. +- `managed.roles`, rendering `spec.managed.roles` so the operator creates and + reconciles a role per application. Together with `databases` this lets every + service connect as the owner of its own database, with no workload holding + superuser rights. +- Render-time validation: a database listed under `databases` that duplicates + `bootstrap.database`, a database with no name or owner, and a managed role with + neither `passwordSecret` nor `disablePassword`. + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Adapted from an existing CloudNativePG chart, with the +deployment-specific and GCP-specific parts removed and the remainder made +configurable. + +### Added +- CNPG `Cluster`, plus optional Barman Cloud `ObjectStore` and + `ScheduledBackup`, and an optional ESO `ExternalSecret` for owner credentials. +- Depends on the `oan-common` library chart for names, labels and the mandatory + `resources` contract. +- Object store provider is selectable — `s3` (default), `gcs` or `azure` — with + the URI scheme derived from the provider and credentials passed through + verbatim as the matching `*Credentials` field. S3 defaults to + `inheritFromIAMRole`, so no keys are stored anywhere. +- Render-time validation of provider, bucket/destination, credential keys, + retention policy format, and logical-import configuration. +- `enableSuperuserAccess` (default false), `superuserSecret` and + `primaryUpdateStrategy` exposed. +- `extraClusterSpec` escape hatch for CNPG fields not yet exposed. +- Example per-environment values file for the registry database. + +### Fixed +- `nodeSelector` and `tolerations` now render inside `affinity`, where CNPG + expects them. Previously they were emitted at the top level of the `Cluster` + spec, which is not part of the CRD schema — the API server silently pruned + them, so pod scheduling constraints were never applied. +- `postgresUID` and `postgresGID` are now declared in `values.yaml`. The + templates referenced them but no schema defined them, so they could not be set + without an undocumented override. +- `bootstrap` is omitted entirely when nothing under it is configured, instead of + rendering a null `initdb`. +- `bootstrap.import.source.dbname` defaults to the single entry in + `import.databases` rather than rendering an empty string. + +### Changed +- `cluster.resources` moved to top-level `resources`, so the shared + `oan-common.resources` helper and its "requests and limits are mandatory" + guardrail apply. +- `cluster.instances`, `cluster.storage`, `cluster.walStorage`, + `cluster.postgresql`, `cluster.affinity`, `cluster.monitoring` and + `cluster.bootstrap` flattened to the top level; the `cluster` wrapper is gone. +- `cluster.name` replaced by the conventional `nameOverride` / + `fullnameOverride` pair. +- Backup ServiceAccount annotations moved from `backup.serviceAccountAnnotations` + to `serviceAccount.annotations`, since IRSA is not backup-specific. +- `ScheduledBackup`'s `immediate` and `backupOwnerReference` are configurable + rather than hardcoded. + +### Removed +- GCS-only object store, including the hardcoded `gs://` scheme, + `gkeEnvironment: true` and the GKE Workload Identity service account + annotation example. +- The `premium-rwo-retain` GKE storage class default; `storageClass` now defaults + to empty (the cluster's default class). +- References to the previous deployment's bundle layout, per-bundle values files, + Docker Hardened Images, and named example databases and users. +- Bitnami-specific framing of the logical import, which is now documented as a + generic migration from any existing PostgreSQL, and the `TODO-CONFIRM` markers + left in the original. +- `app.kubernetes.io/part-of: cloudnative-pg` label, replaced by the standard OAN + labels from `oan-common`. diff --git a/charts/postgresql-cnpg/Chart.yaml b/charts/postgresql-cnpg/Chart.yaml new file mode 100644 index 0000000..80b8352 --- /dev/null +++ b/charts/postgresql-cnpg/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: postgresql-cnpg +description: >- + CloudNativePG-managed PostgreSQL cluster for OAN services (Cluster + + optional ScheduledBackup and Barman Cloud ObjectStore). One release per + database. Requires the CloudNativePG operator; backups additionally require + the Barman Cloud Plugin, both cluster-wide installs. +type: application +version: 0.2.1 +# The PostgreSQL major line this chart targets by default. Unlike other OAN +# charts, this is not an image tag: when image.repository is empty the operator +# supplies its own default image. Pin image.* per environment for reproducibility. +appVersion: "17" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - postgresql + - postgres + - cloudnative-pg + - cnpg +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://cloudnative-pg.io/ +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/postgresql-cnpg/README.md b/charts/postgresql-cnpg/README.md new file mode 100644 index 0000000..eb46ab5 --- /dev/null +++ b/charts/postgresql-cnpg/README.md @@ -0,0 +1,220 @@ +# postgresql-cnpg + +A [CloudNativePG](https://cloudnative-pg.io/)-managed PostgreSQL cluster for OAN +services. One Helm release per database. + +This chart renders CNPG custom resources — a `Cluster`, and optionally an +`ObjectStore` and `ScheduledBackup` for backups. It does **not** render a +StatefulSet: the operator owns the pods, failover, and switchover. It keeps the +name `postgresql-cnpg` rather than an `oan-` prefix because it deploys +third-party software rather than an OAN-authored service. + +## Prerequisites + +Neither is installed by this chart, and both are cluster-wide: + +1. **The CloudNativePG operator.** Without it the rendered `Cluster` is an + unrecognised resource and nothing happens. +2. **The Barman Cloud Plugin** — only for backups. Leave `backup.enabled` false + until its CRDs are present. + +> **Status in OAN:** as of this chart's first release, neither is installed in +> `infra-automation`, and no IRSA role exists for the backup bucket. The S3 +> bucket does exist (`oan-dev-pg-backup`, versioned, 30-day expiry), and its +> Terraform comment notes the IRSA role is deliberately deferred until the +> Postgres operator was chosen — this chart is that choice. Until the operator +> and role land, this chart renders and lints but cannot be usefully installed. + +## Install + +```bash +helm dependency update charts/postgresql-cnpg +helm install registry-db charts/postgresql-cnpg \ + -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml +``` + +See [`examples/registry-db.dev.yaml`](./examples/registry-db.dev.yaml) for a +complete, commented per-environment values file. + +## Databases and roles + +A CNPG cluster's `bootstrap.initdb` creates **exactly one** database. Everything +else is a `Database` object the operator reconciles, so one cluster can host +several service databases without any of them sharing tables or credentials. + +```yaml +# the first database, created when the cluster is bootstrapped +bootstrap: + database: registry + owner: registry + ownerSecret: registry-db-app + +# roles for the others, created and reconciled by the operator +managed: + roles: + - name: keycloak + passwordSecret: + name: keycloak-db # Secret with `username` and `password` + +# the others +databases: + - name: keycloak + owner: keycloak +``` + +That renders `Cluster/registry-db` plus `Database/registry-db-keycloak`, giving: + +| Database | Owner | Created by | +|---|---|---| +| `registry` | `registry` | `bootstrap.initdb` | +| `keycloak` | `keycloak` | `Database` object | + +**The point of the separation:** every application connects as the owner of its +own database, so no workload needs superuser rights, and one service cannot read +another's tables. `enableSuperuserAccess` stays at the CNPG default of `false`. + +> **Requires CNPG >= 1.25**, where the `Database` CRD was introduced. On an older +> operator these objects are ignored *silently* — no database, and no error. If a +> database does not appear, check the operator version first. + +Each entry also supports `ensure` (`present`/`absent`), `databaseReclaimPolicy` +(defaults to `retain`, so deleting the object leaves the data), `encoding`, +`locale`, `template`, `allowConnections`, `connectionLimit`, `extensions` and +`schemas`. + +The render fails if you list the bootstrap database under `databases` — the +operator would be asked to create one that already exists — or if a database +names an owner with no `passwordSecret` and no `disablePassword`. + +## Connecting to it + +CNPG creates three services from the cluster name: + +| Service | Points at | +|---|---| +| `-rw` | The primary. Read-write. What applications normally use. | +| `-ro` | Replicas only. Read-only. | +| `-r` | Any instance, primary included. Read-only workloads that tolerate the primary. | + +**Set `fullnameOverride`.** The cluster name becomes the DNS other services +depend on, and renaming a cluster later means recreating it. With +`fullnameOverride: registry-db` the primary is +`registry-db-rw..svc.cluster.local:5432`; without it you get +`-postgresql-cnpg-rw`. + +## Credentials + +The owner password can come from three places: + +1. **CNPG generates it** — leave `bootstrap.ownerSecret` empty. Read it from + `Secret/-app`. Fine for local and dev. +2. **A Secret you create** — set `bootstrap.ownerSecret` to its name. It must + carry `username` and `password` keys. +3. **A Secret managed by whatever handles secrets in that environment** — point + `bootstrap.ownerSecret` at it. This chart renders no Secrets; it only + references them. + +Superuser access is off by default, matching the CNPG default: applications +connect as the database owner, not as `postgres`. + +## Backups + +Base backups plus continuous WAL archiving through the Barman Cloud Plugin, +which together give point-in-time recovery. Off by default. + +To enable, in this order: + +1. Install the Barman Cloud Plugin cluster-wide. +2. Set `backup.objectStore.bucket` (and `path` if several clusters share the + bucket). +3. Grant access. On EKS that is IRSA: an IAM role trusted by the cluster's + ServiceAccount, named in `serviceAccount.annotations` as + `eks.amazonaws.com/role-arn`. The default credentials block is + `{inheritFromIAMRole: true}`, so no keys go anywhere near the chart. +4. Set `backup.enabled: true`. + +`backup.retentionPolicy` must be **shorter** than any expiry the bucket's own +lifecycle policy applies. The OAN dev bucket expires objects after 30 days, so +the chart's `14d` default sits safely inside it. Get this backwards and S3 +deletes backups Barman still believes it has. + +`provider` supports `s3`, `gcs` and `azure`; `credentials` is passed through +verbatim as the matching `s3Credentials` / `googleCredentials` / +`azureCredentials` field. Because Helm merges maps, switching provider does not +remove the S3 default — the render fails and tells you to null it: + +```bash +--set backup.objectStore.provider=gcs \ +--set backup.objectStore.credentials.inheritFromIAMRole=null \ +--set backup.objectStore.credentials.gkeEnvironment=true +``` + +## Storage + +`storageClass` empty means the cluster's default class. On the OAN clusters that +is `gp3` with **`reclaimPolicy: Delete`**, which deletes the underlying EBS +volume when the PVC goes away. For any database you would miss, point +`storage.storageClass` and `walStorage.storageClass` at a Retain class. + +`walStorage` is a separate volume for the write-ahead log, enabled by default. +It is effectively required for healthy PITR throughput. + +## Migrating an existing database in + +`bootstrap.import` runs CNPG's logical import (`pg_dump`/`pg_restore`) against an +existing PostgreSQL when the cluster is **first created**. It has no effect on an +existing cluster, so it is a one-shot cutover tool, not a sync. + +```yaml +bootstrap: + database: registry + owner: registry + import: + enabled: true + databases: [registry] + source: + host: old-postgres.example + passwordSecret: + name: old-postgres-superuser +``` + +`source.dbname` defaults to the single entry in `databases`. + +## Render-time validation + +The chart fails the render rather than letting a misconfiguration reach the +cluster: + +| Condition | Why it matters | +|---|---| +| `resources` empty | Every OAN component must declare requests and limits (`oan-common.resources`) | +| `backup.enabled` with no `bucket` or `destinationPath` | The ObjectStore would have nowhere to write | +| `backup.objectStore.provider` not `s3`/`gcs`/`azure` | Would render an unknown credentials field | +| A credentials key not valid for the chosen provider | Catches the Helm map-merge trap described above | +| `retentionPolicy` not matching `^[1-9][0-9]*[dwm]$` | The CRD rejects it; `"14days"` fails here instead of at apply | +| `import.enabled` without `source.host` or `source.passwordSecret.name` | CNPG could not read the source | +| `import.databases` not exactly one entry | `type: microservice` permits one database | + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema. CNPG fields the +chart does not expose yet can be passed through `extraClusterSpec`, which is +merged into the `Cluster` spec. + +Note on `appVersion`: for this chart it documents the PostgreSQL major line +targeted by default rather than a pullable tag, because leaving `image.repository` +empty lets the operator supply its own default image. Pin `image.*` per +environment — a PostgreSQL major upgrade is a data migration, not a tag bump. + +## Known gaps + +- **No restore path.** `bootstrap.recovery` (restoring a new cluster from an + object store) is not implemented. Backups are only half a disaster-recovery + story; this needs its own change once backups are actually running. +- **No PodMonitor by default.** `monitoring.enablePodMonitor` requires the + Prometheus Operator CRDs, which are not installed yet. + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/postgresql-cnpg/ci/lint-values.yaml b/charts/postgresql-cnpg/ci/lint-values.yaml new file mode 100644 index 0000000..05874c6 --- /dev/null +++ b/charts/postgresql-cnpg/ci/lint-values.yaml @@ -0,0 +1,10 @@ +# Exercises the backup and ESO paths in CI, which are off in the defaults. +fullnameOverride: ci-db +bootstrap: + database: ci + owner: ci +backup: + enabled: true + objectStore: + bucket: ci-bucket + path: /ci diff --git a/charts/postgresql-cnpg/examples/discovery-db.dev.yaml b/charts/postgresql-cnpg/examples/discovery-db.dev.yaml new file mode 100644 index 0000000..e876898 --- /dev/null +++ b/charts/postgresql-cnpg/examples/discovery-db.dev.yaml @@ -0,0 +1,139 @@ +# Example: the discovery service's database, dev environment. +# +# helm install discovery-db charts/postgresql-cnpg \ +# -n oan-discovery -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml +# +# The discovery service then connects to: +# discovery-db-rw.oan-discovery.svc.cluster.local:5432 +# +# Separate from registry-db in every way - its own cluster, its own namespace, +# its own PostgreSQL major. Nothing here affects the registry stack. + +fullnameOverride: discovery-db + +# --------------------------------------------------------------------------- +# Image +# +# discovery-service needs pgvector, which the stock CloudNativePG operand image +# does not carry. This is the Docker Hardened Image of pgvector: pgvector 0.8.6 +# and pg_trgm 1.6 on PostgreSQL 16, published for both amd64 and arm64 (the OAN +# nodes are arm64). +# +# pg16 rather than the pg14 tag, because PostgreSQL 16 is what the service is +# actually verified against: its README and design doc name it, its +# testcontainers suite pins pgvector/pgvector:0.8.0-pg16, and every measured +# planner claim in the design doc - the 20 jsonpath shapes GIN captures, the +# index selection over 300k rows - was measured on 16. tests/dbtest also uses +# EXPLAIN (GENERIC_PLAN), which is PostgreSQL 16 and later only. +# +# The schema itself does apply cleanly to 14, so this is about staying on the +# major the service was tested on rather than about a hard incompatibility. +# +# CAVEAT, and it is the one to smoke-test first: this is a general-purpose +# PostgreSQL image, not a CNPG operand image. It satisfies CNPG's documented +# requirements - initdb, postgres, pg_ctl, pg_controldata, pg_basebackup and du +# are all on PATH, and CNPG overrides the image's own entrypoint - but it is +# not built from cloudnative-pg/postgres-containers and has not been run under +# the operator here. Verify the first cluster reaches "Cluster in healthy +# state" before pointing anything at it. +# +# It also ships no barman-cloud. That costs nothing while backups run through +# the Barman Cloud Plugin, which supplies its own sidecar, and nothing at all +# while backup.enabled is false - but the in-core backup method is not an +# option with this image. +image: + registry: dhi.io + repository: pgvector + tag: "0.8-pg16" + +# The image's postgres user is uid/gid 70. CNPG defaults to 26 and applies it +# as the pod's runAsUser and fsGroup, so leaving these unset gives the data +# directory to a user that does not exist in this image. +postgresUID: 70 +postgresGID: 70 + +# Dev: single instance. Use 2+ where an outage matters. +instances: 1 + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +storage: + size: 20Gi + # Empty uses the cluster default (gp3, reclaimPolicy Delete). Point this at a + # Retain class before this database holds anything worth keeping. + storageClass: "" + +walStorage: + enabled: true + size: 2Gi + +bootstrap: + database: discovery + owner: discovery + # Omit ownerSecret and CNPG generates Secret/discovery-db-app itself, with a + # ready-made `uri` key the discovery chart consumes through + # database.urlSecret. Simplest correct thing for dev - nothing to create by + # hand and no password to escape. + ownerSecret: "" + + # --------------------------------------------------------------------------- + # This is load-bearing, not a convenience. + # + # discovery-service's first migration opens with: + # CREATE EXTENSION IF NOT EXISTS vector; + # CREATE EXTENSION IF NOT EXISTS pg_trgm; + # + # pg_trgm is marked `trusted = true`, so the database owner may create it. + # `vector` is NOT trusted, and creating it requires superuser: + # + # ERROR: permission denied to create extension "vector" + # HINT: Must be superuser to create this extension. + # + # The service connects as `discovery`, the bootstrap owner, and + # enableSuperuserAccess is false below - so with these lines absent, the very + # first statement of the migration fails and the pod crashloops on an error + # that reads like a bad password rather than a missing grant. + # + # CNPG runs postInitApplicationSQL as superuser inside the freshly created + # application database, which is the one moment superuser is available + # without granting it to anything long-lived. Afterwards the service's own + # `IF NOT EXISTS` is a no-op and the rest of the migration runs as the owner. + # + # Verified against dhi.io/pgvector: as the non-superuser owner the migration + # fails on statement one without these, and applies in full with them. + # --------------------------------------------------------------------------- + postInitApplicationSQL: + - CREATE EXTENSION IF NOT EXISTS vector + - CREATE EXTENSION IF NOT EXISTS pg_trgm + +# The service owns its own database, so nothing here needs superuser rights +# beyond the bootstrap moment above. +enableSuperuserAccess: false + +# One database, one owner - no managed.roles and no extra `databases` entries. +# Unlike registry-db, nothing else shares this cluster. + +# Off until the Barman Cloud Plugin is installed and an IRSA role exists. +# Note the image ships no barman-cloud of its own, so the plugin - which brings +# its own sidecar - is the only backup route here. +backup: + enabled: false + retentionPolicy: "14d" + objectStore: + bucket: oan-dev-pg-backup + path: /discovery + credentials: + inheritFromIAMRole: true + +serviceAccount: + annotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/oan-dev-pg-backup + +commonLabels: + oan.in/environment: dev diff --git a/charts/postgresql-cnpg/examples/registry-db.dev.yaml b/charts/postgresql-cnpg/examples/registry-db.dev.yaml new file mode 100644 index 0000000..2f950d0 --- /dev/null +++ b/charts/postgresql-cnpg/examples/registry-db.dev.yaml @@ -0,0 +1,100 @@ +# Example: the registry service's database, dev environment. +# +# helm install registry-db charts/postgresql-cnpg \ +# -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml +# +# Other workloads then connect to: +# registry-db-rw.oan-registry.svc.cluster.local:5432 + +fullnameOverride: registry-db + +# Sunbird RC's registry is verified against PostgreSQL 14. Pin it rather than +# inheriting the operator default, so an operator upgrade cannot move the major. +image: + registry: ghcr.io + repository: cloudnative-pg/postgresql + tag: "14.13" + +# Dev: single instance. Use 2+ where an outage matters. +instances: 1 + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +storage: + size: 10Gi + # Empty uses the cluster default (gp3, reclaimPolicy Delete). Point this at a + # Retain class before this database holds anything worth keeping. + storageClass: "" + +walStorage: + enabled: true + size: 2Gi + +# The registry's database, created by initdb when the cluster is bootstrapped. +# A cluster gets exactly one of these; everything else goes under `databases`. +bootstrap: + database: registry + owner: registry + # Secret carrying `username` and `password` for the owner role. + # kubectl -n oan-registry create secret generic registry-db-app \ + # --from-literal=username=registry --from-literal=password='' + # Or drop ownerSecret and let CNPG generate the password itself. + ownerSecret: registry-db-app + +# Keycloak's own role. The operator creates and reconciles it, so Keycloak never +# needs superuser rights. +# kubectl -n oan-registry create secret generic keycloak-db \ +# --from-literal=username=keycloak --from-literal=password='' +managed: + roles: + - name: keycloak + passwordSecret: + name: keycloak-db + +# Keycloak's own database, owned by that role. Keeping its realm tables out of +# the registry's database is the one place this deliberately improves on the +# compose stack, which shares a single `registry` database between both services. +# +# REQUIRES CNPG >= 1.25 for the Database CRD. +databases: + - name: keycloak + owner: keycloak + +# Left at the CNPG default (false). Each service connects as the owner of its own +# database, so nothing here needs superuser rights. +# +# The migration chart is the exception: it wants rights across every database it +# migrates. Set this true if you enable migration targets that actually have SQL. +enableSuperuserAccess: false + +# Off until the Barman Cloud Plugin is installed and the IRSA role for +# oan-dev-pg-backup exists (see infra-automation: buckets.pg_backup). +backup: + enabled: false + retentionPolicy: "14d" + objectStore: + bucket: oan-dev-pg-backup + path: /registry + credentials: + inheritFromIAMRole: true + +# The IRSA role the cluster pods assume in order to write those backups. +serviceAccount: + annotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/oan-dev-pg-backup + +# The owner Secret above is created out of band. Until a secrets manager is +# wired up, either create it by hand: +# kubectl -n oan-registry create secret generic registry-db-app \ +# --from-literal=username=registry --from-literal=password='' +# or drop bootstrap.ownerSecret entirely and let CNPG generate the password into +# Secret/registry-db-app itself, which is simpler for dev. + +commonLabels: + oan.in/environment: dev diff --git a/charts/postgresql-cnpg/templates/NOTES.txt b/charts/postgresql-cnpg/templates/NOTES.txt new file mode 100644 index 0000000..16c304f --- /dev/null +++ b/charts/postgresql-cnpg/templates/NOTES.txt @@ -0,0 +1,41 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +CloudNativePG resources in namespace "{{ include "postgresql-cnpg.namespace" . }}": + - Cluster/{{ include "postgresql-cnpg.fullname" . }} ({{ .Values.instances }} instance(s)) +{{- if .Values.backup.enabled }} + - ObjectStore/{{ include "postgresql-cnpg.objectStoreName" . }} -> {{ include "postgresql-cnpg.objectStore.destinationPath" . }} + - ScheduledBackup/{{ include "postgresql-cnpg.fullname" . }}-scheduled ({{ .Values.backup.schedule }}, retention {{ .Values.backup.retentionPolicy }}) +{{- end }} + +Connect from another workload in the cluster: + host: {{ include "postgresql-cnpg.fullname" . }}-rw.{{ include "postgresql-cnpg.namespace" . }}.svc.cluster.local # primary, read-write + host: {{ include "postgresql-cnpg.fullname" . }}-ro.{{ include "postgresql-cnpg.namespace" . }}.svc.cluster.local # replicas, read-only + port: 5432 +{{- with .Values.bootstrap.database }} + database: {{ . }} +{{- end }} +{{- with .Values.bootstrap.owner }} + user: {{ . }} +{{- end }} + +{{ if not .Values.fullnameOverride }} +NOTE: fullnameOverride is not set, so the cluster is named +"{{ include "postgresql-cnpg.fullname" . }}" and other services must connect to +"{{ include "postgresql-cnpg.fullname" . }}-rw". Set fullnameOverride (e.g. +"registry-db") for a shorter, stable name - renaming later means recreating the +cluster. +{{- end }} +{{- if not .Values.bootstrap.ownerSecret }} + +NOTE: bootstrap.ownerSecret is not set, so CNPG generated the owner password +itself. Read it with: + kubectl -n {{ include "postgresql-cnpg.namespace" . }} get secret {{ include "postgresql-cnpg.fullname" . }}-app -o jsonpath='{.data.password}' | base64 -d +{{- end }} +{{- if not .Values.backup.enabled }} + +NOTE: backups are disabled. Enable backup.* once the Barman Cloud Plugin is +installed cluster-wide and the bucket and IRSA role exist. +{{- end }} + +Watch the cluster come up: + kubectl -n {{ include "postgresql-cnpg.namespace" . }} get cluster {{ include "postgresql-cnpg.fullname" . }} -w diff --git a/charts/postgresql-cnpg/templates/_helpers.tpl b/charts/postgresql-cnpg/templates/_helpers.tpl new file mode 100644 index 0000000..901143c --- /dev/null +++ b/charts/postgresql-cnpg/templates/_helpers.tpl @@ -0,0 +1,110 @@ +{{/* +# ============================================================================ +# OAN POSTGRESQL CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the CNPG-specific +# naming and object-store logic this chart needs. +# ============================================================================ +*/}} + +{{/* +Cluster name. Drives the CNPG service names -rw / -ro / -r, +which other charts connect to, so it should be short and stable - set +fullnameOverride (e.g. "registry-db") rather than relying on -. +*/}} +{{- define "postgresql-cnpg.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "postgresql-cnpg.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "postgresql-cnpg.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{/* +Namespace for the CNPG resources. Defaults to the release namespace. +*/}} +{{- define "postgresql-cnpg.namespace" -}} +{{- default (include "oan-common.namespace" .) .Values.namespace -}} +{{- end }} + +{{/* +Image reference, or empty when no image is configured - in which case the +Cluster omits imageName and the operator uses its own default image. +*/}} +{{- define "postgresql-cnpg.image" -}} +{{- if .Values.image.repository -}} +{{- include "oan-common.image" . -}} +{{- end -}} +{{- end }} + +{{/* +Barman ObjectStore name (defaults to -backup). +*/}} +{{- define "postgresql-cnpg.objectStoreName" -}} +{{- default (printf "%s-backup" (include "postgresql-cnpg.fullname" .)) .Values.backup.objectStore.name -}} +{{- end }} + +{{/* +Map the object store provider to its CNPG credentials field. +*/}} +{{- define "postgresql-cnpg.objectStore.credentialsField" -}} +{{- $field := dict "s3" "s3Credentials" "gcs" "googleCredentials" "azure" "azureCredentials" -}} +{{- index $field .Values.backup.objectStore.provider -}} +{{- end }} + +{{/* +Object store destination path. An explicit destinationPath wins; otherwise it is +built from the provider's URI scheme plus bucket and path. +*/}} +{{- define "postgresql-cnpg.objectStore.destinationPath" -}} +{{- $os := .Values.backup.objectStore -}} +{{- if $os.destinationPath -}} +{{- $os.destinationPath -}} +{{- else -}} +{{- $scheme := index (dict "s3" "s3://" "gcs" "gs://" "azure" "azure://") $os.provider -}} +{{- printf "%s%s%s" $scheme $os.bucket ($os.path | default "") -}} +{{- end -}} +{{- end }} + +{{/* +Validate the backup configuration. Called from the templates that need it so a +misconfiguration fails the render rather than producing an ObjectStore the +plugin silently cannot use. +*/}} +{{- define "postgresql-cnpg.validateBackup" -}} +{{- $os := .Values.backup.objectStore -}} +{{- $providers := list "s3" "gcs" "azure" -}} +{{- if not (has $os.provider $providers) -}} +{{- fail (printf "%s: backup.objectStore.provider must be one of %s, got %q" .Chart.Name (join ", " $providers) $os.provider) -}} +{{- end -}} +{{- if and (not $os.bucket) (not $os.destinationPath) -}} +{{- fail (printf "%s: backup.enabled is true but neither backup.objectStore.bucket nor backup.objectStore.destinationPath is set" .Chart.Name) -}} +{{- end -}} +{{- if not $os.credentials -}} +{{- fail (printf "%s: backup.enabled is true but backup.objectStore.credentials is empty. For S3 with IRSA use {inheritFromIAMRole: true}" .Chart.Name) -}} +{{- end -}} +{{/* +Credentials keys must belong to the chosen provider. Helm merges maps, so +switching provider away from the s3 default would otherwise carry +`inheritFromIAMRole` into googleCredentials, where the plugin rejects it. +*/}} +{{- $allowed := index (dict + "s3" (list "accessKeyId" "inheritFromIAMRole" "region" "secretAccessKey" "sessionToken") + "gcs" (list "applicationCredentials" "gkeEnvironment") + "azure" (list "connectionString" "inheritFromAzureAD" "storageAccount" "storageKey" "storageSasToken" "useDefaultAzureCredentials") + ) $os.provider -}} +{{- range $key, $_ := $os.credentials -}} +{{- if not (has $key $allowed) -}} +{{- fail (printf "%s: backup.objectStore.credentials.%s is not valid for provider %q (allowed: %s). If it came from this chart's default, null it out: --set backup.objectStore.credentials.%s=null" $.Chart.Name $key $os.provider (join ", " $allowed) $key) -}} +{{- end -}} +{{- end -}} +{{- with .Values.backup.retentionPolicy -}} +{{- if not (regexMatch "^[1-9][0-9]*[dwm]$" .) -}} +{{- fail (printf "backup.retentionPolicy must match ^[1-9][0-9]*[dwm]$ (days, weeks or months, e.g. \"14d\"), got %q" .) -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/charts/postgresql-cnpg/templates/cluster.yaml b/charts/postgresql-cnpg/templates/cluster.yaml new file mode 100644 index 0000000..01d1123 --- /dev/null +++ b/charts/postgresql-cnpg/templates/cluster.yaml @@ -0,0 +1,202 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ include "postgresql-cnpg.fullname" . }} + namespace: {{ include "postgresql-cnpg.namespace" . }} + labels: + {{- include "postgresql-cnpg.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + instances: {{ .Values.instances }} + {{- with (include "postgresql-cnpg.image" .) }} + imageName: {{ . | quote }} + {{- end }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.postgresUID }} + postgresUID: {{ . }} + {{- end }} + {{- with .Values.postgresGID }} + postgresGID: {{ . }} + {{- end }} + primaryUpdateStrategy: {{ .Values.primaryUpdateStrategy }} + enableSuperuserAccess: {{ .Values.enableSuperuserAccess }} + {{- with .Values.superuserSecret }} + superuserSecret: + name: {{ . }} + {{- end }} + + {{- with .Values.serviceAccount.annotations }} + serviceAccountTemplate: + metadata: + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + + storage: + size: {{ .Values.storage.size }} + {{- with .Values.storage.storageClass }} + storageClass: {{ . }} + {{- end }} + {{- if .Values.walStorage.enabled }} + walStorage: + size: {{ .Values.walStorage.size }} + {{- with .Values.walStorage.storageClass }} + storageClass: {{ . }} + {{- end }} + {{- end }} + + resources: + {{- include "oan-common.resources" . | nindent 4 }} + + {{- $affinity := omit .Values.affinity "nodeSelector" "tolerations" }} + {{- with .Values.affinity.nodeSelector }} + {{- $affinity = set $affinity "nodeSelector" . }} + {{- end }} + {{- with .Values.affinity.tolerations }} + {{- $affinity = set $affinity "tolerations" . }} + {{- end }} + {{- with $affinity }} + # CNPG nests nodeSelector and tolerations inside affinity; the Cluster spec + # has no top-level fields for them, and anything put there is silently + # dropped by the API server. + affinity: + {{- toYaml . | nindent 4 }} + {{- end }} + + {{- with .Values.managed.roles }} + {{/* + Roles created and reconciled by the operator. A database owner must exist + before the Database object referencing it can be applied, so any role named as + an owner under `databases` belongs here. + */}} + managed: + roles: + {{- range . }} + {{- if not .name }} + {{- fail (printf "%s: every entry in managed.roles needs a name" $.Chart.Name) }} + {{- end }} + - name: {{ .name | quote }} + ensure: {{ .ensure | default "present" }} + login: {{ .login | default true }} + {{- if hasKey . "superuser" }} + superuser: {{ .superuser }} + {{- end }} + {{- if hasKey . "createdb" }} + createdb: {{ .createdb }} + {{- end }} + {{- if hasKey . "createrole" }} + createrole: {{ .createrole }} + {{- end }} + {{- if hasKey . "inherit" }} + inherit: {{ .inherit }} + {{- end }} + {{- with .connectionLimit }} + connectionLimit: {{ . }} + {{- end }} + {{- with .inRoles }} + inRoles: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .comment }} + comment: {{ . | quote }} + {{- end }} + {{- if .passwordSecret }} + passwordSecret: + name: {{ .passwordSecret.name }} + {{- else if .disablePassword }} + disablePassword: true + {{- else }} + {{- fail (printf "%s: managed role %q has no passwordSecret. Set one (a Secret with username and password keys), or set disablePassword: true if the role should not log in with a password." $.Chart.Name .name) }} + {{- end }} + {{- end }} + {{- end }} + + {{- with .Values.postgresql.parameters }} + postgresql: + parameters: + {{- toYaml . | nindent 6 }} + {{- end }} + + {{- if .Values.monitoring.enablePodMonitor }} + monitoring: + enablePodMonitor: true + {{- end }} + + {{- if .Values.backup.enabled }} + {{- include "postgresql-cnpg.validateBackup" . }} + # Base backups and WAL archiving via the Barman Cloud Plugin (CNPG-I). + plugins: + - name: {{ .Values.backup.pluginName }} + isWALArchiver: true + parameters: + barmanObjectName: {{ include "postgresql-cnpg.objectStoreName" . }} + {{- end }} + + {{- $boot := .Values.bootstrap }} + {{- if or $boot.database $boot.owner $boot.ownerSecret $boot.postInitApplicationSQL $boot.import.enabled }} + bootstrap: + initdb: + {{- with .Values.bootstrap.database }} + database: {{ . | quote }} + {{- end }} + {{- with .Values.bootstrap.owner }} + owner: {{ . | quote }} + {{- end }} + {{- with .Values.bootstrap.ownerSecret }} + secret: + name: {{ . }} + {{- end }} + {{- if .Values.bootstrap.import.enabled }} + import: + type: {{ .Values.bootstrap.import.type }} + databases: + {{- toYaml .Values.bootstrap.import.databases | nindent 10 }} + source: + externalCluster: {{ include "postgresql-cnpg.fullname" . }}-source + {{- with .Values.bootstrap.postInitApplicationSQL }} + postImportApplicationSQL: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- else }} + {{- with .Values.bootstrap.postInitApplicationSQL }} + postInitApplicationSQL: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} + + {{- if .Values.bootstrap.import.enabled }} + {{- $src := .Values.bootstrap.import.source }} + {{- if not $src.host }} + {{- fail (printf "%s: bootstrap.import.enabled is true but bootstrap.import.source.host is empty" .Chart.Name) }} + {{- end }} + {{- if not (eq (len .Values.bootstrap.import.databases) 1) }} + {{- fail (printf "%s: bootstrap.import.type microservice needs exactly one entry in bootstrap.import.databases, got %d" .Chart.Name (len .Values.bootstrap.import.databases)) }} + {{- end }} + {{- if not $src.passwordSecret.name }} + {{- fail (printf "%s: bootstrap.import.enabled is true but bootstrap.import.source.passwordSecret.name is empty - CNPG needs credentials to read the source" .Chart.Name) }} + {{- end }} + # The existing PostgreSQL this cluster is imported from, read logically at + # creation time only. + externalClusters: + - name: {{ include "postgresql-cnpg.fullname" . }}-source + connectionParameters: + host: {{ $src.host | quote }} + port: {{ $src.port | quote }} + dbname: {{ $src.dbname | default (first .Values.bootstrap.import.databases) | quote }} + user: {{ $src.user | quote }} + password: + name: {{ $src.passwordSecret.name }} + key: {{ $src.passwordSecret.key }} + {{- end }} + {{- with .Values.extraClusterSpec }} + {{- toYaml . | nindent 2 }} + {{- end }} diff --git a/charts/postgresql-cnpg/templates/database.yaml b/charts/postgresql-cnpg/templates/database.yaml new file mode 100644 index 0000000..896845b --- /dev/null +++ b/charts/postgresql-cnpg/templates/database.yaml @@ -0,0 +1,64 @@ +{{- range .Values.databases }} +{{- if not .name }} +{{- fail (printf "%s: every entry in databases needs a name" $.Chart.Name) }} +{{- end }} +{{- if not .owner }} +{{- fail (printf "%s: database %q needs an owner. The role must exist - declare it under managed.roles, or use the cluster's bootstrap owner." $.Chart.Name .name) }} +{{- end }} +{{- if eq .name ($.Values.bootstrap.database | default "") }} +{{- fail (printf "%s: database %q is already created by bootstrap.database. Remove it from `databases`, or rename one of them - the operator would otherwise be asked to create a database that already exists." $.Chart.Name .name) }} +{{- end }} +--- +{{/* +A database created declaratively by the CNPG operator, alongside the single +database that bootstrap.initdb creates. + +Requires CNPG >= 1.25, which is where the Database CRD was introduced. On an +older operator this object is simply ignored - no database, no error - so check +the operator version if a database does not appear. +*/}} +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: {{ printf "%s-%s" (include "postgresql-cnpg.fullname" $) .name | trunc 63 | trimSuffix "-" }} + namespace: {{ include "postgresql-cnpg.namespace" $ }} + labels: + {{- include "postgresql-cnpg.labels" $ | nindent 4 }} + oan.in/database: {{ .name | quote }} + {{- with $.Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + cluster: + name: {{ include "postgresql-cnpg.fullname" $ }} + name: {{ .name | quote }} + owner: {{ .owner | quote }} + ensure: {{ .ensure | default "present" }} + # retain: dropping this object leaves the database in place. Databases hold + # data that outlives a Helm release, so deletion is deliberately not automatic. + databaseReclaimPolicy: {{ .databaseReclaimPolicy | default "retain" }} + {{- with .encoding }} + encoding: {{ . | quote }} + {{- end }} + {{- with .locale }} + locale: {{ . | quote }} + {{- end }} + {{- with .template }} + template: {{ . | quote }} + {{- end }} + {{- if hasKey . "allowConnections" }} + allowConnections: {{ .allowConnections }} + {{- end }} + {{- with .connectionLimit }} + connectionLimit: {{ . }} + {{- end }} + {{- with .extensions }} + extensions: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .schemas }} + schemas: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/postgresql-cnpg/templates/objectstore.yaml b/charts/postgresql-cnpg/templates/objectstore.yaml new file mode 100644 index 0000000..bf2a86f --- /dev/null +++ b/charts/postgresql-cnpg/templates/objectstore.yaml @@ -0,0 +1,41 @@ +{{- if .Values.backup.enabled }} +{{- include "postgresql-cnpg.validateBackup" . }} +{{- $os := .Values.backup.objectStore }} +# Barman Cloud Plugin ObjectStore - target for base backups and WAL archiving. +# Requires the Barman Cloud Plugin installed cluster-wide. +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: {{ include "postgresql-cnpg.objectStoreName" . }} + namespace: {{ include "postgresql-cnpg.namespace" . }} + labels: + {{- include "postgresql-cnpg.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.backup.retentionPolicy }} + retentionPolicy: {{ . | quote }} + {{- end }} + configuration: + destinationPath: {{ include "postgresql-cnpg.objectStore.destinationPath" . | quote }} + {{- with $os.endpointURL }} + endpointURL: {{ . | quote }} + {{- end }} + {{- with $os.serverName }} + serverName: {{ . | quote }} + {{- end }} + {{ include "postgresql-cnpg.objectStore.credentialsField" . }}: + {{- toYaml $os.credentials | nindent 6 }} + wal: + compression: {{ $os.wal.compression }} + {{- with $os.wal.encryption }} + encryption: {{ . }} + {{- end }} + data: + compression: {{ $os.data.compression }} + {{- with $os.data.encryption }} + encryption: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/postgresql-cnpg/templates/scheduledbackup.yaml b/charts/postgresql-cnpg/templates/scheduledbackup.yaml new file mode 100644 index 0000000..252b2ee --- /dev/null +++ b/charts/postgresql-cnpg/templates/scheduledbackup.yaml @@ -0,0 +1,27 @@ +{{- if .Values.backup.enabled }} +# Scheduled base backups. WAL is archived continuously by the plugin, so these +# plus the archived WAL are what make point-in-time recovery possible. +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: {{ include "postgresql-cnpg.fullname" . }}-scheduled + namespace: {{ include "postgresql-cnpg.namespace" . }} + labels: + {{- include "postgresql-cnpg.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + # 6-field cron: sec min hour dom mon dow + schedule: {{ .Values.backup.schedule | quote }} + backupOwnerReference: {{ .Values.backup.backupOwnerReference }} + immediate: {{ .Values.backup.immediate }} + method: plugin + pluginConfiguration: + name: {{ .Values.backup.pluginName }} + parameters: + barmanObjectName: {{ include "postgresql-cnpg.objectStoreName" . }} + cluster: + name: {{ include "postgresql-cnpg.fullname" . }} +{{- end }} diff --git a/charts/postgresql-cnpg/values.yaml b/charts/postgresql-cnpg/values.yaml new file mode 100644 index 0000000..68c3958 --- /dev/null +++ b/charts/postgresql-cnpg/values.yaml @@ -0,0 +1,271 @@ +# ============================================================================ +# postgresql-cnpg - default values +# +# One CloudNativePG `Cluster` (plus optional backups) per database. Install one +# release per service that needs its own database, giving each its own +# credentials, storage and backup schedule: +# +# helm install registry-db charts/postgresql-cnpg \ +# -n oan-registry -f values/registry-db.dev.yaml +# +# PREREQUISITES (neither is installed by this chart): +# 1. The CloudNativePG operator, cluster-wide. +# 2. For backups only: the Barman Cloud Plugin, also cluster-wide. Leave +# backup.enabled false until its CRDs are present. +# +# Minimum you must set for a real deployment: +# - fullnameOverride (see below - it decides the DNS other services use) +# - bootstrap.database / bootstrap.owner / bootstrap.ownerSecret +# - storage.size and storage.storageClass +# ============================================================================ + +# Namespace for the CNPG resources. Defaults to the release namespace; set this +# only if you deliberately want them somewhere other than `helm -n`. +namespace: "" + +nameOverride: "" +# STRONGLY RECOMMENDED. The cluster name becomes the connection DNS other +# services use - -rw (primary), -ro (replicas), -r (any). +# Without this you get -postgresql-cnpg-rw. Set something short and +# stable, e.g. "registry-db" -> registry-db-rw..svc.cluster.local +fullnameOverride: "" + +# --------------------------------------------------------------------------- +# Image +# Leave repository empty to let the CNPG operator choose its own default +# PostgreSQL image. Pin it per environment for reproducible upgrades - a +# PostgreSQL major version change is a data migration, not a tag bump. +# --------------------------------------------------------------------------- +image: + # e.g. "ghcr.io" + registry: dhi.io + # e.g. "cloudnative-pg/postgresql" + repository: pgvector + tag: "0.8-pg16" + # sha256:... - wins over tag when set + digest: "" + pullSecrets: [] + +# Primary + (instances - 1) replicas. 1 is fine for local or dev; use at least +# 2 anywhere an outage matters. +instances: 2 + +# Resource requests and limits for each instance. REQUIRED - the render fails +# when this is empty (oan-common.resources). +resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +# --------------------------------------------------------------------------- +# Storage +# storageClass empty means the cluster's default class. Note the OAN clusters +# default to gp3 with reclaimPolicy: Delete - for anything you cannot lose, +# point this at a Retain class so the volume survives PVC deletion. +# --------------------------------------------------------------------------- +storage: + size: 10Gi + storageClass: "" + +# Separate volume for the write-ahead log. Recommended, and effectively +# required for healthy point-in-time-recovery throughput. +walStorage: + enabled: true + size: 2Gi + storageClass: "" + +# PostgreSQL server parameters, merged onto the CNPG defaults. +postgresql: + parameters: {} + # max_connections: "200" + # shared_buffers: "256MB" + +# Pod scheduling. NOTE: in CNPG, nodeSelector and tolerations belong *inside* +# affinity - the Cluster spec has no top-level fields for them, and anything +# put there is silently dropped by the API server. +affinity: + enablePodAntiAffinity: true + topologyKey: kubernetes.io/hostname + nodeSelector: {} + tolerations: [] + +# Annotations for the ServiceAccount CNPG creates for the cluster pods. This is +# where IRSA goes when backups write to S3: +# eks.amazonaws.com/role-arn: arn:aws:iam:::role/ +serviceAccount: + annotations: {} + +# Prometheus PodMonitor. Requires the Prometheus Operator CRDs. +monitoring: + enablePodMonitor: false + +# Superuser access is disabled by default (CNPG default too): applications +# should connect as the database owner, not as postgres. +enableSuperuserAccess: false +# Name of a Secret with the superuser credentials. Only used when +# enableSuperuserAccess is true; CNPG generates one if left empty. +superuserSecret: "" + +# UID/GID the postgres process runs as. Empty uses the operator default (26). +postgresUID: "" +postgresGID: "" + +# How replicas are updated: "unsupervised" rolls automatically, +# "supervised" waits for a manual switchover. +primaryUpdateStrategy: unsupervised + +# --------------------------------------------------------------------------- +# Bootstrap: how the database is first created. +# --------------------------------------------------------------------------- +bootstrap: + # Application database and its owner. Both empty uses the CNPG defaults + # ("app"/"app"), which is rarely what you want - name them explicitly. + database: "" + owner: "" + # Name of a Secret with keys `username` and `password` for the owner. Leave + # empty to have CNPG generate the password into Secret/-app. + ownerSecret: "" + # SQL run once, after the database is created. + postInitApplicationSQL: [] + # - CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + + # Logical import from an existing PostgreSQL, for migrating an existing + # database into CNPG. CNPG runs pg_dump/pg_restore against `source` when the + # cluster is created. One-time: it has no effect on an existing cluster. + import: + enabled: false + # microservice = one database per cluster (databases must list exactly one) + type: microservice + databases: [] + source: + host: "" + port: "5432" + dbname: "" + user: postgres + # Secret holding the source user's password + passwordSecret: + name: "" + key: password + +# --------------------------------------------------------------------------- +# Additional databases +# +# bootstrap.initdb creates exactly ONE database. Everything else goes here, as +# CNPG `Database` objects the operator reconciles. +# +# REQUIRES CNPG >= 1.25 (where the Database CRD was introduced). On an older +# operator these objects are ignored silently - no database, and no error. +# +# The owner must be a role that exists. Declare it under managed.roles below, or +# use the cluster's bootstrap owner. +# +# databases: +# - name: keycloak +# owner: keycloak +# extensions: +# - name: uuid-ossp +# +# Each entry supports: name, owner, ensure (present|absent), +# databaseReclaimPolicy (retain|delete), encoding, locale, template, +# allowConnections, connectionLimit, extensions, schemas. +# +# Listing the bootstrap database here fails the render - the operator would be +# asked to create a database that already exists. +# --------------------------------------------------------------------------- +databases: [] + +# --------------------------------------------------------------------------- +# Managed roles +# +# Roles the operator creates and reconciles. Give every application its own role +# owning its own database, so no workload needs superuser rights. +# +# passwordSecret names a Secret with `username` and `password` keys. The operator +# reconciles the role's password to match it. +# +# managed: +# roles: +# - name: keycloak +# passwordSecret: +# name: keycloak-db +# +# A role with neither passwordSecret nor disablePassword fails the render, since +# it would be a login role nobody can authenticate as. +# --------------------------------------------------------------------------- +managed: + roles: [] + +# --------------------------------------------------------------------------- +# Backups: base backups + continuous WAL archiving via the Barman Cloud Plugin. +# +# Disabled by default. Enabling it requires, in this order: +# 1. The Barman Cloud Plugin installed cluster-wide. +# 2. A bucket that the cluster can write to. +# 3. Credentials - on EKS, an IRSA role bound to the cluster ServiceAccount +# via serviceAccount.annotations above. +# --------------------------------------------------------------------------- +backup: + enabled: false + pluginName: barman-cloud.cloudnative-pg.io + + # Must be shorter than any expiry the bucket's own lifecycle policy applies, + # or the bucket deletes backups Barman still counts on. Form: d|w|m. + retentionPolicy: "14d" + + # CNPG uses 6-field cron (sec min hour dom mon dow). Default: daily at 02:00. + schedule: "0 0 2 * * *" + # Take a backup as soon as the ScheduledBackup is created. + immediate: true + backupOwnerReference: self + + objectStore: + # Defaults to -backup + name: "" + # s3 | gcs | azure + provider: s3 + # Bucket name only - the URI scheme comes from provider. + # OAN dev provisions "oan-dev-pg-backup" (see infra-automation). + bucket: "" + # Optional prefix so several clusters can share a bucket, e.g. "/registry" + path: "" + # Full destination path, overriding bucket/path. For S3-compatible stores. + destinationPath: "" + # For S3-compatible object stores (MinIO and similar) + endpointURL: "" + # Pins the folder inside the bucket. Defaults to the cluster name; set it + # explicitly before renaming a cluster that already has backups. + serverName: "" + + # Rendered verbatim as Credentials. The default is IRSA: the + # cluster's ServiceAccount carries the role, no keys anywhere. + # + # S3 with static keys instead (each value is a secret reference): + # credentials: + # accessKeyId: {name: pg-backup-creds, key: ACCESS_KEY_ID} + # secretAccessKey: {name: pg-backup-creds, key: SECRET_ACCESS_KEY} + # GCS on GKE: credentials: {gkeEnvironment: true} + # Azure workload identity: credentials: {inheritFromAzureAD: true} + credentials: + inheritFromIAMRole: true + + wal: + # bzip2 | gzip | lz4 | snappy | xz | zstd + compression: gzip + # AES256 | aws:kms. Empty uses the bucket's default encryption. + encryption: "" + data: + # bzip2 | gzip | lz4 | snappy + compression: gzip + encryption: "" + +# Escape hatch: extra keys merged into the Cluster spec, for CNPG fields this +# chart does not expose yet. Keys the chart already sets cannot be overridden +# here - that would render duplicate YAML keys. +extraClusterSpec: {} + +# Extra labels and annotations on every resource this chart renders +commonLabels: {} +commonAnnotations: {} diff --git a/charts/postgresql-migration/.helmignore b/charts/postgresql-migration/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/postgresql-migration/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/postgresql-migration/CHANGELOG.md b/charts/postgresql-migration/CHANGELOG.md new file mode 100644 index 0000000..05c805d --- /dev/null +++ b/charts/postgresql-migration/CHANGELOG.md @@ -0,0 +1,104 @@ +# Changelog + +All notable changes to the `postgresql-migration` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Removed +- The `00-bootstrap` target and its `CREATE DATABASE` migration. Databases are now + created by the `postgresql-cnpg` chart, via `bootstrap.database` and the CNPG + `Database` CRD, which the operator reconciles properly. Doing it in Flyway meant + `CREATE DATABASE` outside a transaction (with a `.sql.conf` sidecar), no + `IF NOT EXISTS`, and a standing rule never to list the database the cluster + itself creates. Leaving both in place would have failed outright, since the + operator creates the database first. + + This chart now only migrates schemas inside existing databases, which is what + Flyway is for. + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Adapted from an existing Flyway migration chart, rebuilt on +`oan-common` with the previous deployment's specifics removed. + +### Added +- Flyway migration Job, env ConfigMap, generated-script ConfigMap, one ConfigMap + per target, ServiceAccount, and an optional ESO ExternalSecret. +- Multi-target migrations: each entry in `targets` is one database plus its + migration directory, applied in the declared order. `enabled: false` skips an + entry without removing it. +- A bootstrap migration that creates the `keycloak` database, shipping the + `.sql.conf` sidecar with `executeInTransaction=false` that PostgreSQL requires + for `CREATE DATABASE`. +- Runs as a Helm hook (`pre-install`, `pre-upgrade`) by default, so migrations + complete before dependent services start. `hook.enabled: false` installs it as + an ordinary release resource instead. +- `repairOnFailure` (default true) runs `flyway repair` and retries once. +- Render-time validation of `postgresql.host` and + `postgresql.passwordSecret.name`, and of every target: name and database + present, no duplicate names, and a matching directory in the chart. +- Targets with no `.sql` files are skipped at runtime with a log line, so a + database whose schema is managed elsewhere (Keycloak's Liquibase, Sunbird RC's + own DDL) can still be listed for inventory. +- `activeDeadlineSeconds` (default 900), so an unreachable database fails the Job + rather than blocking `helm install` until Helm's timeout. + +### Fixed +- **The database password is no longer written to a ConfigMap.** The original + chart rendered `FLYWAY_PASSWORD` and `PGPASSWORD` into ConfigMap data via its + `configs/env.yaml`, which put a live credential in plaintext in an object that + is not treated as sensitive. It is now injected from a `secretKeyRef`, and the + chart renders no passwords at all. +- Volume mounts for migration directories are now inside the block that emits + the `volumeMounts:` key. Previously the per-folder `range` sat outside the + `if configmap.enabled` guard, so disabling the ConfigMap while keeping + migration folders emitted orphaned list items with no parent key — invalid + YAML. The same applied to `volumes:`. +- `FLYWAY_LOCATIONS` no longer disagrees with the actual mount path. The original + set it to `filesystem:/flyway/migrations` while the script passed + `-locations=filesystem:/migrations/$folder` explicitly and the volumes mounted + at `/migrations/`, so the environment variable was misleading dead + configuration. +- Target ordering is explicit in `targets` rather than derived from + `ls /migrations | sort -n`, which silently depended on directory names carrying + numeric prefixes. + +### Changed +- Built on `oan-common`, replacing the external `common` library chart pulled + from a third-party Helm repository. +- The migration script is generated from `targets` rather than shipped as a + static file that parsed a directory listing, so each target gets its own JDBC + URL and one Job can migrate several databases. +- Script is POSIX `sh`, not `bash`, so it runs on the Alpine-based Flyway image. +- Image tag pinned (`11.10.0-alpine`) rather than defaulting to `latest`. +- Resource requests and limits are now mandatory, via `oan-common.resources`. + +### Removed +- `Service`, `Ingress`, `autoscaling`/HPA, `serviceMonitor`, and liveness and + readiness probes. This chart renders a Job; none of them apply to one. +- `replicaCount`, which a Job does not have. +- Configuration belonging to the previous deployment and unrelated to database + migration: `superset_oauth_clientid`, `superset_oauth_client_secret`, + `kong_ingress_domain`, `gf_auth_generic_oauth_client_id`, + `gf_auth_generic_oauth_client_secret`, `web_console_user`, + `web_console_password`, `web_console_login`, and the `system_settings` block + (`encryption_key`, `default_dataset_id`, `max_event_size`, `dedup_period`). +- The `global.postgresql.password` value, along with `global.*` indirection and + the `base.namespace` override mechanism that let a parent chart retarget the + namespace per subchart. +- The discovery-specific migrations (`02-discover-db`), which belong to a + different service and a different database. diff --git a/charts/postgresql-migration/Chart.yaml b/charts/postgresql-migration/Chart.yaml new file mode 100644 index 0000000..938a0b7 --- /dev/null +++ b/charts/postgresql-migration/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +name: postgresql-migration +description: >- + Flyway schema migrations for OAN PostgreSQL databases, run as a Job. Creates + the per-service databases and applies versioned SQL to each. Runs as a Helm + hook so it completes before the services that depend on it start. +type: application +version: 0.2.0 +# Flyway CLI image tag +appVersion: "11.10.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - postgresql + - flyway + - migration +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://documentation.red-gate.com/flyway +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/postgresql-migration/README.md b/charts/postgresql-migration/README.md new file mode 100644 index 0000000..adf2a78 --- /dev/null +++ b/charts/postgresql-migration/README.md @@ -0,0 +1,160 @@ +# postgresql-migration + +Flyway schema migrations for OAN PostgreSQL databases, run as a Kubernetes Job. + +It does two things, in the order you declare: + +1. **Creates the per-service databases** the cluster does not create itself. +2. **Applies versioned SQL** to each database. + +By default it runs as a Helm hook, so `helm install` waits for migrations to +finish before the services that depend on them start. + +## Install + +```bash +helm dependency update charts/postgresql-migration +helm install migrate charts/postgresql-migration \ + -n oan-registry -f charts/postgresql-migration/examples/registry-stack.dev.yaml +``` + +Run it **after** the database cluster and **before** Keycloak — Keycloak needs the +`keycloak` database to exist. See +[`charts/registry/README.md`](../registry/README.md) for the full stack order. + +## Targets + +A target is one database plus the migration directory applied to it. They run in +the order listed in `targets`: + +```yaml +targets: + - name: 01-registry # directory under files/migrations/ + database: registry # database to connect to (must already exist) + - name: 02-keycloak + database: keycloak +``` + +The list order is what runs; directory names are numbered only so they read in +the same order on disk. A target whose directory holds no `.sql` files is +**skipped**, and the Job logs that it skipped it — so a database can be listed +for inventory purposes without inventing migrations for it. + +Set `enabled: false` on an entry to skip it without deleting it. + +## This chart does not create databases + +Databases are created by the [`postgresql-cnpg`](../postgresql-cnpg) chart — +`bootstrap.database` for the first, and `databases` (CNPG `Database` objects) for +the rest. The operator reconciles them. + +That is deliberate. Creating them here would mean `CREATE DATABASE` outside a +transaction, no `IF NOT EXISTS` to make it idempotent, and a standing rule never +to list the database the cluster already made. The operator does it properly, and +Flyway goes back to what it is for: **migrating schemas inside databases that +already exist.** + +So a target here names a database that must already exist. Pointing one at a +missing database fails with a connection error, not a helpful message. + +## Adding a migration## Adding a migration + +Drop a file into the target's directory, following Flyway's naming +(`V__.sql`): + +``` +charts/postgresql-migration/files/migrations/01-registry/V1__add_lookup_index.sql +``` + +Then bump the chart version and add a CHANGELOG entry. Two rules that Flyway +enforces and this chart cannot soften for you: + +- **Never edit a migration that has already run.** Flyway stores a checksum; + changing the file makes the next run fail validation. Add a new version + instead. +- **Versions must not collide.** Two `V1__` files in the same target is an error. + +Keycloak's own tables are the exception to all of this: it manages its schema +with Liquibase, and a second tool writing those tables would fight it. Do not add +Keycloak table DDL here. + +## The connection + +One host, one role, many databases — each target reuses the connection with a +different database name. + +The role needs: + +- **CREATEDB** for the bootstrap target. +- **Ownership or superuser rights** on each database it migrates. + +In the OAN dev layout that is the CNPG superuser, whose password CNPG generates +into `Secret/-superuser` when the database chart sets +`enableSuperuserAccess: true`. + +`postgresql.host` must be the **primary** (`-rw`). Migrations write, so +a read-only replica service fails — and fails in a way that reads like a +permissions problem. + +## Secrets + +The password is injected as `FLYWAY_PASSWORD` from a `secretKeyRef`. It is +**never** written into a ConfigMap — that was a real flaw in the chart this was +adapted from, where it landed in ConfigMap data in plaintext. + +This chart renders no Secrets; the one it references is created by whatever +manages secrets in that environment. In the OAN dev layout CNPG generates it +(`Secret/-superuser`). + +## Running as a hook, or not + +| | `hook.enabled: true` (default) | `hook.enabled: false` | +|---|---|---| +| When it runs | During `helm install`/`helm upgrade`, before other resources | As an ordinary release resource | +| Helm waits for it | Yes | No | +| `helm uninstall` removes it | No — `deletePolicy` handles cleanup | Yes | +| Repeated upgrades | Works: `before-hook-creation` deletes the old Job first | Fails if the pod template changed — Jobs are immutable | + +`hook.weight: "-5"` orders this ahead of any other hook that needs the schema. +The database cluster must already be running when the hook fires, which holds in +the OAN layout because the cluster is a separate, earlier release. + +## Failure handling + +`repairOnFailure: true` runs `flyway repair` and retries once. Repair rewrites +the schema history to match the migrations on disk, which fixes the common case — +a previous run that failed partway and left a checksum mismatch — but it will +also happily paper over a migration someone edited. Fine for dev; consider +turning it off where you want a failed migration to stay failed until someone +looks at it. + +The Job's `backoffLimit` retries the pod. That is safe: Flyway skips +already-applied migrations. + +`activeDeadlineSeconds` (default 900) stops a Job that cannot reach the database +from blocking `helm install` until Helm's own timeout. + +## Reading what it did + +```bash +kubectl -n oan-registry logs job/migrate-postgresql-migration +``` + +The log names each target, the URL, the migration files it found, and whether it +skipped, succeeded or failed. Per-database history lives in +`flyway_schema_history` in each database. + +Note that with the default hook `deletePolicy: before-hook-creation`, the Job and +its logs survive until the next install or upgrade replaces it. + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema and +[`examples/registry-stack.dev.yaml`](./examples/registry-stack.dev.yaml) for a +per-environment file. + +## Versioning + +Every change — **including adding or editing a migration** — needs a `version` +bump in `Chart.yaml` and an entry in [`CHANGELOG.md`](./CHANGELOG.md). See +[`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/postgresql-migration/ci/lint-values.yaml b/charts/postgresql-migration/ci/lint-values.yaml new file mode 100644 index 0000000..fcf4a69 --- /dev/null +++ b/charts/postgresql-migration/ci/lint-values.yaml @@ -0,0 +1,7 @@ +# Minimum values that let `helm template` run in CI. This chart deliberately +# fails the render when the host and password Secret are unset, so bare defaults +# cannot be templated. Dummy references only - see examples/ for real config. +postgresql: + host: registry-db-rw + passwordSecret: + name: registry-db-superuser diff --git a/charts/postgresql-migration/ci/no-hook-values.yaml b/charts/postgresql-migration/ci/no-hook-values.yaml new file mode 100644 index 0000000..d19c047 --- /dev/null +++ b/charts/postgresql-migration/ci/no-hook-values.yaml @@ -0,0 +1,11 @@ +# Second CI case: the Job as an ordinary release resource rather than a Helm +# hook, which is how you would run migrations on their own. +postgresql: + host: registry-db-rw + passwordSecret: + name: registry-db-superuser +hook: + enabled: false +job: + ttlSecondsAfterFinished: 3600 +repairOnFailure: false diff --git a/charts/postgresql-migration/examples/registry-stack.dev.yaml b/charts/postgresql-migration/examples/registry-stack.dev.yaml new file mode 100644 index 0000000..b5880f2 --- /dev/null +++ b/charts/postgresql-migration/examples/registry-stack.dev.yaml @@ -0,0 +1,55 @@ +# Example: migrations for the registry stack, dev environment. +# +# helm install migrate charts/postgresql-migration \ +# -n oan-registry -f charts/postgresql-migration/examples/registry-stack.dev.yaml +# +# NOTE: with both databases now created by the postgresql-cnpg chart, and with +# Sunbird RC and Keycloak both managing their own schemas, this chart currently +# has nothing to do for the registry stack - every target below is empty and gets +# skipped. It is here for when OAN adds schemas of its own (catalog publish and +# discovery, per engineering-tracker #30), so the targets and their history exist +# from the start rather than being retrofitted. +# +# Run it after the database cluster. + +postgresql: + # The CNPG primary. Migrations write, so this must not be a replica. + host: registry-db-rw + port: 5432 + # Needs rights on each database it migrates. The CNPG superuser has them + # everywhere; a per-database owner would need one release per database. + # + # NOTE: the dev database example leaves enableSuperuserAccess false, since no + # application needs it. Turn it on there before enabling any target that + # actually has SQL, or point this at a per-database owner instead. + user: postgres + passwordSecret: + name: registry-db-superuser + key: password + +targets: + # No .sql files yet: Sunbird RC creates its own tables. The target is kept so + # anything OAN adds alongside the registry has a versioned home. + - name: 01-registry + database: registry + + # No .sql files, and there should not be: Keycloak manages its own schema with + # Liquibase. Listed so the database appears in this chart's inventory. + - name: 02-keycloak + database: keycloak + +# Dev convenience. Consider turning this off in production, so a failed +# migration stays failed until someone reads the log rather than being repaired +# automatically. +repairOnFailure: true + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +commonLabels: + oan.in/environment: dev diff --git a/charts/postgresql-migration/files/migrations/01-registry/README.md b/charts/postgresql-migration/files/migrations/01-registry/README.md new file mode 100644 index 0000000..bd90952 --- /dev/null +++ b/charts/postgresql-migration/files/migrations/01-registry/README.md @@ -0,0 +1,13 @@ +# registry migrations + +Versioned SQL applied to the `registry` database, in Flyway naming order +(`V1__…`, `V2__…`). + +**This directory is intentionally empty of migrations.** Sunbird RC core creates +its own tables from the entity definitions it is given, so the registry needs no +Flyway-managed schema today. The target exists so that anything OAN adds +alongside the registry — a view, an index, a lookup table — has an obvious home +and a versioned history. + +A target whose directory contains no `.sql` files is skipped, and the Job logs +that it skipped it. diff --git a/charts/postgresql-migration/files/migrations/02-keycloak/README.md b/charts/postgresql-migration/files/migrations/02-keycloak/README.md new file mode 100644 index 0000000..8d8f291 --- /dev/null +++ b/charts/postgresql-migration/files/migrations/02-keycloak/README.md @@ -0,0 +1,10 @@ +# keycloak migrations + +Versioned SQL applied to the `keycloak` database. + +**This directory is intentionally empty of migrations.** Keycloak manages its own +schema with Liquibase on startup, and a second migration tool writing the same +tables would fight it. Do not add table DDL for Keycloak's own tables here. + +The target exists only so the database appears in this chart's inventory. A +target whose directory contains no `.sql` files is skipped. diff --git a/charts/postgresql-migration/templates/NOTES.txt b/charts/postgresql-migration/templates/NOTES.txt new file mode 100644 index 0000000..c015735 --- /dev/null +++ b/charts/postgresql-migration/templates/NOTES.txt @@ -0,0 +1,41 @@ +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +{{ if .Values.hook.enabled -}} +The migration Job runs as a Helm hook ({{ join ", " .Values.hook.events }}), so it +has already completed - `helm install` would have failed otherwise. +{{- else -}} +The migration Job is an ordinary release resource, so it is starting now. +{{- end }} + +Targets, in order: +{{- range $targets }} + - {{ .name }} -> database {{ .database }}{{ if not (include "postgresql-migration.targetHasSql" (dict "ctx" $ "name" .name)) }} (no .sql files - will be skipped){{ end }} +{{- end }} + +Connection: {{ .Values.postgresql.user }}@{{ .Values.postgresql.host }}:{{ .Values.postgresql.port }} + +Read what it did: + kubectl -n {{ .Release.Namespace }} logs job/{{ include "postgresql-migration.fullname" . }} + +{{ if .Values.hook.enabled -}} +Because the Job is a hook with deletePolicy "{{ .Values.hook.deletePolicy }}", it +stays until the next install or upgrade replaces it - so the logs above are +readable now, but will be gone after the next run. +{{- end }} + +Check the schema history Flyway keeps, per database: +{{- range $targets }} +{{- if include "postgresql-migration.targetHasSql" (dict "ctx" $ "name" .name) }} + psql -h {{ $.Values.postgresql.host }} -U {{ $.Values.postgresql.user }} -d {{ .database }} \ + -c 'SELECT version, description, success, installed_on FROM flyway_schema_history ORDER BY installed_rank' +{{- end }} +{{- end }} + +If a target failed, the log names which one. The usual causes: + - The role lacks CREATEDB (bootstrap target) or rights on the database. + - A database listed in the bootstrap migration already exists, created by the + PostgreSQL cluster itself. Remove it from the migration - the cluster's + bootstrap database must not be created here. + - A migration was edited after it ran; Flyway rejects the checksum change. + {{ if .Values.repairOnFailure }}repairOnFailure is on, so it retried once after `flyway repair`.{{ else }}repairOnFailure is off, so it failed without retrying.{{ end }} diff --git a/charts/postgresql-migration/templates/_helpers.tpl b/charts/postgresql-migration/templates/_helpers.tpl new file mode 100644 index 0000000..d8bbb21 --- /dev/null +++ b/charts/postgresql-migration/templates/_helpers.tpl @@ -0,0 +1,146 @@ +{{/* +# ============================================================================ +# POSTGRESQL-MIGRATION CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the Flyway target +# and JDBC wiring this chart needs. +# ============================================================================ +*/}} + +{{- define "postgresql-migration.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "postgresql-migration.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "postgresql-migration.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "postgresql-migration.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "postgresql-migration.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "postgresql-migration.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "postgresql-migration.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +Targets that are enabled, in the order given. `enabled` defaults to true when +the key is absent, so a target can be written as just name + database. +*/}} +{{- define "postgresql-migration.enabledTargets" -}} +{{- $out := list -}} +{{- range .Values.targets -}} +{{- if or (not (hasKey . "enabled")) .enabled -}} +{{- $out = append $out . -}} +{{- end -}} +{{- end -}} +{{- $out | toJson -}} +{{- end }} + +{{/* +Validate the connection and every target. Called from the Job and the ConfigMap +so a misconfiguration fails the render rather than the Job. +*/}} +{{- define "postgresql-migration.validate" -}} +{{- $pg := .Values.postgresql -}} +{{- if not $pg.host -}} +{{- fail (printf "%s: postgresql.host is required - point it at the PostgreSQL PRIMARY service, e.g. registry-db-rw. Migrations write, so a read-only replica will not do." .Chart.Name) -}} +{{- end -}} +{{- if not $pg.passwordSecret.name -}} +{{- fail (printf "%s: postgresql.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) -}} +{{- end -}} +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray -}} +{{- if not $targets -}} +{{- fail (printf "%s: no enabled entries in targets - there is nothing to migrate" .Chart.Name) -}} +{{- end -}} +{{- $seen := dict -}} +{{- range $targets -}} +{{- if not .name -}} +{{- fail (printf "%s: every entry in targets needs a name, matching a directory under files/migrations/" $.Chart.Name) -}} +{{- end -}} +{{- if not .database -}} +{{- fail (printf "%s: target %q needs a database" $.Chart.Name .name) -}} +{{- end -}} +{{- if hasKey $seen .name -}} +{{- fail (printf "%s: target %q is listed twice" $.Chart.Name .name) -}} +{{- end -}} +{{- $seen = set $seen .name true -}} +{{- $glob := printf "files/migrations/%s/*" .name -}} +{{- if not ($.Files.Glob $glob) -}} +{{- fail (printf "%s: target %q has no directory files/migrations/%s in the chart" $.Chart.Name .name .name) -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +JDBC URL for one database. +*/}} +{{- define "postgresql-migration.jdbcUrl" -}} +{{- $pg := .ctx.Values.postgresql -}} +{{- printf "jdbc:postgresql://%s:%v/%s%s" $pg.host $pg.port .database $pg.jdbcParams -}} +{{- end }} + +{{/* +Name of the ConfigMap holding the migration files for one target. +*/}} +{{- define "postgresql-migration.targetConfigMapName" -}} +{{- printf "%s-%s" (include "postgresql-migration.fullname" .ctx) .name | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Where a target's migrations are mounted. +*/}} +{{- define "postgresql-migration.targetMountPath" -}} +{{- printf "/migrations/%s" .name -}} +{{- end }} + +{{/* +Migration files for one target, as ConfigMap data. + +Only .sql and .conf are included: the directories also hold READMEs explaining +what belongs in them, and those are not migrations. +*/}} +{{- define "postgresql-migration.targetFiles" -}} +{{- $ctx := .ctx -}} +{{- $name := .name -}} +{{- range $path, $_ := $ctx.Files.Glob (printf "files/migrations/%s/*.sql" $name) }} +{{ base $path }}: |- + {{- $ctx.Files.Get $path | nindent 2 }} +{{- end }} +{{- range $path, $_ := $ctx.Files.Glob (printf "files/migrations/%s/*.conf" $name) }} +{{ base $path }}: |- + {{- $ctx.Files.Get $path | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Does this target actually have migrations to apply? +*/}} +{{- define "postgresql-migration.targetHasSql" -}} +{{- if .ctx.Files.Glob (printf "files/migrations/%s/*.sql" .name) -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +Hook annotations, when the Job runs as a Helm hook. +*/}} +{{- define "postgresql-migration.hookAnnotations" -}} +{{- if .Values.hook.enabled }} +"helm.sh/hook": {{ join "," .Values.hook.events | quote }} +"helm.sh/hook-weight": {{ .Values.hook.weight | quote }} +"helm.sh/hook-delete-policy": {{ .Values.hook.deletePolicy | quote }} +{{- end }} +{{- end }} diff --git a/charts/postgresql-migration/templates/configmap-migrations.yaml b/charts/postgresql-migration/templates/configmap-migrations.yaml new file mode 100644 index 0000000..4bfd143 --- /dev/null +++ b/charts/postgresql-migration/templates/configmap-migrations.yaml @@ -0,0 +1,29 @@ +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +{{- range $targets }} +{{- $files := include "postgresql-migration.targetFiles" (dict "ctx" $ "name" .name) | trim }} +--- +{{/* +Migration files for one target. Rendered even when empty, so the mount exists and +the Job can report "no .sql files" rather than "not mounted". +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-migration.targetConfigMapName" (dict "ctx" $ "name" .name) }} + labels: + {{- include "postgresql-migration.labels" $ | nindent 4 }} + oan.in/migration-target: {{ .name | quote }} + annotations: + {{- include "postgresql-migration.hookAnnotations" $ | nindent 4 }} + {{- with $.Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if $files }} +data: + {{- $files | nindent 2 }} +{{- else }} +# No .sql or .conf files in files/migrations/{{ .name }} - this database's schema +# is managed outside Flyway. The Job skips this target. +data: {} +{{- end }} +{{- end }} diff --git a/charts/postgresql-migration/templates/configmap-script.yaml b/charts/postgresql-migration/templates/configmap-script.yaml new file mode 100644 index 0000000..6e148eb --- /dev/null +++ b/charts/postgresql-migration/templates/configmap-script.yaml @@ -0,0 +1,87 @@ +{{- include "postgresql-migration.validate" . }} +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-migration.fullname" . }}-script + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + migrate.sh: |- + #!/bin/sh + # Generated by the postgresql-migration chart. Do not edit in the cluster - + # change values.yaml and upgrade instead. + # + # POSIX sh, not bash: the flyway alpine image has no bash. + # + # One `flyway migrate` per target, in the order declared in values.yaml. + # Flyway records what it has applied in each database's own + # flyway_schema_history, so re-running this is safe: applied migrations are + # skipped, not repeated. + set -eu + + fail=0 + + run_target() { + name="$1" + url="$2" + dir="/migrations/$name" + + echo "==========================================================" + echo "target: $name" + echo "url: $url" + + if [ ! -d "$dir" ]; then + echo " SKIP: $dir is not mounted" + return 0 + fi + # A target with no .sql is a placeholder for a database whose schema is + # managed elsewhere (Keycloak's Liquibase, Sunbird RC's own DDL). + if [ -z "$(find "$dir" -maxdepth 1 -name '*.sql' -print -quit)" ]; then + echo " SKIP: no .sql files in $dir" + return 0 + fi + + echo " migrations:" + for f in "$dir"/*.sql; do + echo " $(basename "$f")" + done + + if flyway -url="$url" -locations="filesystem:$dir" migrate; then + echo " OK: $name" + return 0 + fi + + {{- if .Values.repairOnFailure }} + echo " migrate failed for $name - running repair and retrying once" + if ! flyway -url="$url" -locations="filesystem:$dir" repair; then + echo " FAILED: repair failed for $name" + return 1 + fi + if flyway -url="$url" -locations="filesystem:$dir" migrate; then + echo " OK: $name (after repair)" + return 0 + fi + echo " FAILED: $name still failing after repair" + return 1 + {{- else }} + echo " FAILED: $name (repairOnFailure is disabled, so not retrying)" + return 1 + {{- end }} + } + + {{- range $targets }} + run_target {{ .name | quote }} {{ include "postgresql-migration.jdbcUrl" (dict "ctx" $ "database" .database) | quote }} || fail=1 + {{- end }} + + echo "==========================================================" + if [ "$fail" -ne 0 ]; then + echo "one or more targets FAILED" + exit 1 + fi + echo "all targets completed" diff --git a/charts/postgresql-migration/templates/configmap.yaml b/charts/postgresql-migration/templates/configmap.yaml new file mode 100644 index 0000000..c50910b --- /dev/null +++ b/charts/postgresql-migration/templates/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "postgresql-migration.envConfigMapName" . }} + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + # Flyway settings. The password is NOT here - it comes from a Secret, injected + # as FLYWAY_PASSWORD in the Job. + FLYWAY_USER: {{ .Values.postgresql.user | quote }} + FLYWAY_BASELINE_ON_MIGRATE: {{ .Values.flyway.baselineOnMigrate | quote }} + FLYWAY_BASELINE_VERSION: {{ .Values.flyway.baselineVersion | quote }} + FLYWAY_VALIDATE_ON_MIGRATE: {{ .Values.flyway.validateOnMigrate | quote }} + FLYWAY_OUT_OF_ORDER: {{ .Values.flyway.outOfOrder | quote }} + {{- range $k, $v := .Values.flyway.extraEnv }} + {{ $k }}: {{ $v | quote }} + {{- end }} + {{- range $k, $v := .Values.envConfig }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/charts/postgresql-migration/templates/job.yaml b/charts/postgresql-migration/templates/job.yaml new file mode 100644 index 0000000..585b954 --- /dev/null +++ b/charts/postgresql-migration/templates/job.yaml @@ -0,0 +1,105 @@ +{{- include "postgresql-migration.validate" . }} +{{- $targets := include "postgresql-migration.enabledTargets" . | fromJsonArray }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "postgresql-migration.fullname" . }} + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + backoffLimit: {{ .Values.job.backoffLimit }} + {{- with .Values.job.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ . }} + {{- end }} + {{- with .Values.job.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ . }} + {{- end }} + template: + metadata: + labels: + {{- include "postgresql-migration.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: {{ .Values.job.restartPolicy }} + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "postgresql-migration.serviceAccountName" . }} + {{- end }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + containers: + - name: flyway + image: {{ include "postgresql-migration.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + command: + - /bin/sh + - /script/migrate.sh + env: + - name: FLYWAY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.passwordSecret.name }} + key: {{ .Values.postgresql.passwordSecret.key }} + {{- with (include "oan-common.env" . | trim) }} + {{- . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ include "postgresql-migration.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + - name: script + mountPath: /script + readOnly: true + {{- range $targets }} + - name: {{ printf "migrations-%s" .name | trunc 63 | trimSuffix "-" }} + mountPath: {{ include "postgresql-migration.targetMountPath" (dict "name" .name) }} + readOnly: true + {{- end }} + volumes: + - name: script + configMap: + name: {{ include "postgresql-migration.fullname" . }}-script + defaultMode: 0555 + {{- range $targets }} + - name: {{ printf "migrations-%s" .name | trunc 63 | trimSuffix "-" }} + configMap: + name: {{ include "postgresql-migration.targetConfigMapName" (dict "ctx" $ "name" .name) }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/postgresql-migration/templates/serviceaccount.yaml b/charts/postgresql-migration/templates/serviceaccount.yaml new file mode 100644 index 0000000..0097ee0 --- /dev/null +++ b/charts/postgresql-migration/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "postgresql-migration.serviceAccountName" . }} + labels: + {{- include "postgresql-migration.labels" . | nindent 4 }} + annotations: + {{- include "postgresql-migration.hookAnnotations" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/postgresql-migration/values.yaml b/charts/postgresql-migration/values.yaml new file mode 100644 index 0000000..d54e215 --- /dev/null +++ b/charts/postgresql-migration/values.yaml @@ -0,0 +1,194 @@ +# ============================================================================ +# postgresql-migration - default values +# +# Flyway migrations for OAN PostgreSQL databases, run as a Job. It does two +# things, in order: +# +# 1. Creates the per-service databases the cluster does not create itself. +# 2. Applies versioned SQL to each database. +# +# It runs as a Helm hook by default, so `helm install` of a service chart waits +# for migrations to finish before the service starts. +# +# Minimum you must set: +# - postgresql.host +# - postgresql.passwordSecret.name +# ============================================================================ + +image: + registry: docker.io + repository: flyway/flyway + # Pinned; `latest` in a migration job means a tool upgrade you did not choose. + tag: "11.10.0-alpine" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + # The Job talks to PostgreSQL, not the Kubernetes API. + automountServiceAccountToken: false + +# --------------------------------------------------------------------------- +# Connection +# +# One host, one role, many databases: each target below reuses this connection +# with a different database name. +# +# The role needs ownership or superuser rights on each database it migrates, in +# order to create objects and the flyway_schema_history table. +# +# This chart does NOT create databases. That is the postgresql-cnpg chart's job - +# `bootstrap.database` for the first one and `databases` for the rest, both +# reconciled by the CNPG operator. Creating them here instead would mean +# CREATE DATABASE outside a transaction, no IF NOT EXISTS, and a rule about never +# listing the database the cluster already made. The operator does it properly. +# --------------------------------------------------------------------------- +postgresql: + # PostgreSQL primary service, e.g. "registry-db-rw". Must be the primary: + # migrations write. + host: "" + port: 5432 + user: postgres + # Secret holding the password. Required - the chart renders no passwords. + passwordSecret: + name: "" + key: password + # Appended to every JDBC URL, e.g. "?sslmode=require" + jdbcParams: "" + +# --------------------------------------------------------------------------- +# Targets +# +# Each entry is one database and the migration directory applied to it, run in +# the order listed. The order in this list is what runs - directory names are +# numbered only so they read in the same order on disk. +# +# name: the directory under files/migrations/ +# database: the database to connect to +# enabled: set false to skip without removing the entry +# +# A target whose directory has no .sql files is skipped, and the Job says so. +# --------------------------------------------------------------------------- +targets: + - name: 01-registry + database: registry + enabled: true + - name: 02-keycloak + database: keycloak + enabled: true + +# Run `flyway repair` and retry once when a migration fails. +# +# Repair rewrites the schema history to match the migrations on disk. It fixes +# the common case - a previous run that failed partway and left a checksum +# mismatch - but it will also happily paper over a genuinely edited migration. +# Leave it on for dev; consider turning it off where you want a failed migration +# to stay failed until someone looks at it. +repairOnFailure: true + +# Flyway's own settings, passed through as environment variables. Anything +# Flyway reads from FLYWAY_* can go here. +flyway: + # Create the schema history table on a database that has objects but no + # history yet, instead of refusing to run. + baselineOnMigrate: true + # Version recorded for that baseline. + baselineVersion: "0" + # Reject a migration whose checksum no longer matches the recorded one. + validateOnMigrate: true + # Allow out-of-order migrations. Off: a lower version arriving late is an error. + outOfOrder: false + # Extra FLYWAY_* environment variables, as name -> value. + extraEnv: {} + # FLYWAY_CONNECT_RETRIES: "10" + +# --------------------------------------------------------------------------- +# Helm hook +# +# As a hook, the Job runs during `helm install`/`helm upgrade` and Helm waits for +# it before proceeding. That is what orders migrations ahead of the services. +# +# Two consequences worth knowing: +# - A hook resource is not tracked in the release, so `helm uninstall` does not +# remove it; deletePolicy handles the cleanup instead. +# - The database cluster must already be running when the hook fires. That +# holds in the OAN layout, where the cluster is a separate, earlier release. +# +# Set enabled: false to install the Job as an ordinary release resource, e.g. to +# run migrations by themselves. +# --------------------------------------------------------------------------- +hook: + enabled: true + events: + - pre-install + - pre-upgrade + # Lower weights run first, so keep this below any other hook that needs the + # schema to exist. + weight: "-5" + # before-hook-creation deletes the previous Job before creating this one, + # which is what makes repeated upgrades work: a Job's pod template is + # immutable, so re-applying a changed one is rejected. + deletePolicy: before-hook-creation + +job: + # Pod-level retries. Flyway is idempotent - already-applied migrations are + # skipped - so a retry is safe. + backoffLimit: 3 + # Give up after this long. Without it a Job that cannot reach the database + # blocks `helm install` until Helm's own timeout. + activeDeadlineSeconds: 900 + # Delete the finished Job (and its pod, and its logs) this long after it ends. + # Empty keeps it until the next run, so the logs are there to read. + ttlSecondsAfterFinished: "" + restartPolicy: OnFailure + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +# Off by default, as elsewhere in this repo. Flyway only reads its migrations and +# writes to the database, so it is a good candidate for tightening - but verify +# the pinned image's own user before enabling runAsUser, since a mismatch shows +# up as a permission error on /flyway rather than anything obvious. +podSecurityContext: + enabled: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# Non-secret extra configuration, injected via envFrom. +envConfig: {} + +# Extra env vars from specific Secret keys. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} From e7dd191b6ff146c450ce8176a8947cf45548df2b Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:35:04 +0530 Subject: [PATCH 52/81] feat: add keycloak chart with sunbird-rc realm import [#53] Imports the sunbird-rc realm from a ConfigMap on first start, with the realm content checksummed into the pod annotations so a realm change rolls the pod. An init container waits for the database, standing in for compose's depends_on: condition: service_healthy. Keycloak connects as the owner of its own keycloak database rather than sharing the registry's database as postgres, which is how the compose stack runs it. The realm JSON is Sunbird RC's export and still carries its upstream defaults: a placeholder admin-api client secret, an enabled placeholder user with a known password, and a wildcard redirect URI on the public frontend client. All three need hardening before this reaches any environment that is not local. See the chart README for the client secret step; the other two are tracked as follow-up. --- charts/keycloak/.helmignore | 13 + charts/keycloak/CHANGELOG.md | 72 + charts/keycloak/Chart.yaml | 28 + charts/keycloak/README.md | 194 ++ charts/keycloak/ci/lint-values.yaml | 12 + charts/keycloak/examples/keycloak.dev.yaml | 77 + charts/keycloak/examples/keycloak.prod.yaml | 97 + charts/keycloak/files/realm-export.json | 2312 +++++++++++++++++ charts/keycloak/templates/NOTES.txt | 36 + charts/keycloak/templates/_helpers.tpl | 159 ++ .../keycloak/templates/configmap-realm.yaml | 27 + charts/keycloak/templates/configmap.yaml | 16 + charts/keycloak/templates/deployment.yaml | 126 + charts/keycloak/templates/ingress.yaml | 41 + .../templates/poddisruptionbudget.yaml | 33 + charts/keycloak/templates/service.yaml | 19 + charts/keycloak/templates/serviceaccount.yaml | 13 + .../templates/tests/test-connection.yaml | 39 + charts/keycloak/values.yaml | 313 +++ 19 files changed, 3627 insertions(+) create mode 100644 charts/keycloak/.helmignore create mode 100644 charts/keycloak/CHANGELOG.md create mode 100644 charts/keycloak/Chart.yaml create mode 100644 charts/keycloak/README.md create mode 100644 charts/keycloak/ci/lint-values.yaml create mode 100644 charts/keycloak/examples/keycloak.dev.yaml create mode 100644 charts/keycloak/examples/keycloak.prod.yaml create mode 100644 charts/keycloak/files/realm-export.json create mode 100644 charts/keycloak/templates/NOTES.txt create mode 100644 charts/keycloak/templates/_helpers.tpl create mode 100644 charts/keycloak/templates/configmap-realm.yaml create mode 100644 charts/keycloak/templates/configmap.yaml create mode 100644 charts/keycloak/templates/deployment.yaml create mode 100644 charts/keycloak/templates/ingress.yaml create mode 100644 charts/keycloak/templates/poddisruptionbudget.yaml create mode 100644 charts/keycloak/templates/service.yaml create mode 100644 charts/keycloak/templates/serviceaccount.yaml create mode 100644 charts/keycloak/templates/tests/test-connection.yaml create mode 100644 charts/keycloak/values.yaml diff --git a/charts/keycloak/.helmignore b/charts/keycloak/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/keycloak/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/keycloak/CHANGELOG.md b/charts/keycloak/CHANGELOG.md new file mode 100644 index 0000000..3977df8 --- /dev/null +++ b/charts/keycloak/CHANGELOG.md @@ -0,0 +1,72 @@ +# Changelog + +All notable changes to the `keycloak` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Configuration ported from the verified `registry/docker-compose.yml` +stack, with the deployment shape taken from Sunbird's own Helm charts +(`deploy-as-code/helm/v2`). + +### Added +- Deployment, Service, env ConfigMap, realm ConfigMap, ServiceAccount, and + optional Ingress and ESO ExternalSecret, built on the `oan-common` library. +- All ten Keycloak environment variables from the compose stack, verified at + parity: `DB_VENDOR`, `DB_ADDR`, `DB_PORT`, `DB_DATABASE`, `DB_USER`, + `DB_PASSWORD`, `KEYCLOAK_USER`, `KEYCLOAK_PASSWORD`, `KEYCLOAK_IMPORT` and + `PROXY_ADDRESS_FORWARDING`. +- The `sunbird-rc` realm export shipped in `files/`, rendered into a ConfigMap + and mounted for `KEYCLOAK_IMPORT`, so `helm install` needs nothing prepared + beforehand. A `checksum/realm` annotation rolls the pod when it changes. +- `realmImport.existingConfigMap` to mount a realm ConfigMap managed outside the + chart instead, and `realmImport.realmJson` to supply one inline. +- Render-time validation that `database.host`, `database.passwordSecret.name` + and `admin.passwordSecret.name` are set, since the chart creates no passwords. +- `strategy: Recreate`, so two instances never run the realm import at once. + +- Init containers that wait for the database before starting, so a first install + does not crashloop while PostgreSQL comes up. Derived from `database.host`. +- Optional PodDisruptionBudget, which fails the render if enabled alongside a + single replica - that combination blocks node drains entirely. +- `helm test` check that `/auth` responds and the imported realm is served. +- `extraVolumes` / `extraVolumeMounts` / `extraInitContainers` escape hatches. +- Per-environment example values for dev and production. The production example + connects as a dedicated `keycloak` role rather than the superuser, pulls the + admin password from ESO, and documents why it stays at one replica: the legacy + WildFly distribution needs Infinispan cache clustering configured before a + second replica can share sessions. + +### Notes +- The shipped realm is a second copy of the compose stack's + `registry/imports/realm-export.json`, byte-identical at the time of writing. + Nothing enforces that: if the compose copy changes, this one needs updating and + a chart version bump. +- Editing the realm rolls the pod but does not re-import it. The legacy image + imports a realm only when it is absent, so changes to a live realm have to be + made in the Keycloak console or against a fresh database. + +### Changed from the compose stack +- Probes follow Sunbird's Helm chart, not the compose healthcheck: readiness and + startup check `/auth/` on 8080, liveness is a TCP check. Compose curls the + WildFly management port 9990, which this chart deliberately does not expose. +- Image tag defaults to `v1.0.0` rather than `latest`, matching the version + Sunbird pairs with `sunbird-rc-core:v2.0.0`. A chart should not deploy a + moving tag. +- Resource limits added. Sunbird's chart sets requests only; limits are + mandatory in this repo. diff --git a/charts/keycloak/Chart.yaml b/charts/keycloak/Chart.yaml new file mode 100644 index 0000000..d93c746 --- /dev/null +++ b/charts/keycloak/Chart.yaml @@ -0,0 +1,28 @@ +apiVersion: v2 +name: keycloak +description: >- + Keycloak for the OAN registry, on the Sunbird RC Keycloak image. Issues and + validates the tokens the registry authorises requests with, and imports the + realm, clients and roles the registry expects. +type: application +version: 0.2.0 +# Tag of ghcr.io/sunbird-rc/sunbird-rc-keycloak. Sunbird's own v2 deployment +# pairs this version with sunbird-rc-core v2.0.0. +appVersion: "v1.0.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - keycloak + - sunbird-rc + - authentication +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/Sunbird-RC/devops/tree/main/deploy-as-code +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/keycloak/README.md b/charts/keycloak/README.md new file mode 100644 index 0000000..d69576e --- /dev/null +++ b/charts/keycloak/README.md @@ -0,0 +1,194 @@ +# keycloak + +Keycloak for the OAN registry, on the [Sunbird RC Keycloak +image](https://github.com/Sunbird-RC/devops/tree/main/deploy-as-code). It issues +and validates the tokens the registry authorises requests with, and imports the +realm, clients and roles the registry expects. + +This is the **legacy (WildFly/JBoss) Keycloak distribution**: it serves under +`/auth` and takes `DB_VENDOR`-style environment variables, not the modern `KC_*` +ones. Charts written for upstream Keycloak will not work with this image, which +is why this chart exists rather than depending on one. + +## What it renders + +| Resource | Notes | +|---|---| +| Deployment | `strategy: Recreate` — one instance, and the realm import runs on startup. Init container waits for the database | +| Service | `ClusterIP` on 8080 | +| ConfigMap (env) | Extra non-secret config from `envConfig` | +| ConfigMap (realm) | The realm shipped in `files/`, mounted for `KEYCLOAK_IMPORT` | +| ServiceAccount | | +| PodDisruptionBudget | Optional, off by default | +| Ingress | Optional, off by default | +| Test Pod | `helm test` check on `/auth` and the imported realm | + +Configuration matches `registry/docker-compose.yml` exactly — all ten Keycloak +environment variables, same names and same values. + +## Install + +```bash +helm dependency update charts/keycloak +helm install keycloak charts/keycloak \ + -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml +``` + +Requires a reachable PostgreSQL — see [`postgresql-cnpg`](../postgresql-cnpg) — +and two Secrets that this chart never creates: the database password and the +Keycloak admin password. + +## The two-phase first install + +**The registry cannot authenticate immediately after this chart installs.** The +realm export ships with the `admin-api` client secret masked (`**********`), +which is not a working credential. So: + +1. Install this chart; it imports the realm on first start. +2. Reach the admin console: + ```bash + kubectl -n oan-registry port-forward svc/keycloak 8080:8080 + ``` + then open `http://localhost:8080/auth/admin`. +3. Realm `sunbird-rc` → Clients → `admin-api` → Credentials → **Regenerate + Secret**. +4. Store that value where the registry chart's + `keycloak.adminClientSecret` points. +5. Install the registry. + +There is no way for a chart to shortcut this while the realm is imported from a +masked export. Making it one-phase means managing clients declaratively instead +— `keycloak-config-cli` or the Keycloak Operator — which is a larger change. + +## Waiting for the database + +Kubernetes has no equivalent of compose's `depends_on: condition: +service_healthy`, so an init container waits for the database before Keycloak +starts: + +```yaml +waitFor: + enabled: true + database: true # host and port come from `database` +``` + +Without it, a first install crashloops while PostgreSQL comes up — which recovers +on its own, but looks broken. + +## Verifying an install + +```bash +helm test keycloak -n oan-registry +``` + +Checks that `/auth` responds and, when `realmImport.enabled`, that the realm is +actually being served. That second check is the one that matters: the registry +validates tokens against that exact realm URL. + +## Replicas and clustering + +**This chart stays at one replica, and that is deliberate.** The legacy WildFly +distribution needs its Infinispan cache cluster configured — JGroups discovery +and session replication — before a second replica is safe. Without it, a token +issued by one replica is not recognised by the other. This chart does not +configure that, so `replicaCount: 1` and `strategy: Recreate` are the honest +settings. + +The practical consequence: restarting Keycloak briefly interrupts token issuing. +The registry keeps serving requests whose tokens are already valid. + +Getting past this means either configuring Infinispan clustering, or moving to a +current Keycloak with the Keycloak Operator — which would also remove the masked +client-secret problem below. + +`podDisruptionBudget` is therefore off by default: over a single pod it blocks +node drains entirely. + +## Per-environment values + +| File | For | +|---|---| +| [`examples/keycloak.dev.yaml`](./examples/keycloak.dev.yaml) | Dev: superuser connection, hand-made Secrets, no ingress | +| [`examples/keycloak.prod.yaml`](./examples/keycloak.prod.yaml) | Production: dedicated `keycloak` role, JVM heap sized to the limit | + +## Realm management + +The chart ships `files/realm-export.json` — realm `sunbird-rc`, 8 clients, +including the `admin-api` and `registry-frontend` clients the registry uses — and +renders it into a ConfigMap mounted at `realmImport.mountPath`. The legacy image +imports it on startup via `KEYCLOAK_IMPORT`. + +That means **`helm install` needs nothing prepared beforehand**, and a realm +change rolls the pod through the `checksum/realm` annotation. + +| You want | Set | +|---|---| +| The shipped realm | nothing — this is the default | +| A realm managed outside the chart | `existingConfigMap: ` (must already exist; the chart renders none) | +| A different realm inline | `realmJson: ` | +| No import at all | `enabled: false` | + +### Two things to know + +**The shipped realm is a second copy.** It is byte-identical to the compose +stack's `registry/imports/realm-export.json`, which is deliberate — the cluster +should import the same realm that was verified locally. But nothing enforces +that: if the compose copy changes, this one has to be updated too, with a chart +version bump. That is the cost of the chart being self-contained. + +**Editing the realm does not re-import it.** The legacy image imports a realm +only when it is *absent*. Changing the JSON rolls the pod, but Keycloak will +leave an already-imported realm alone — so changes to a live realm have to be +made in the Keycloak console, or by starting from a fresh database. This catches +people out, because the pod restart makes it look like something happened. + +## Database## Database + +The chart's `values.yaml` defaults match compose — Keycloak sharing the +registry's database — but **the example values point it at its own `keycloak` +database**, which is how the stack is meant to be deployed. Sunbird's own Helm +charts do the same. Keycloak's realm tables have no reason to sit beside registry +records, and separating them frees the registry to connect as a non-superuser +owner. + +The `keycloak` database is created by +[`postgresql-migration`](../postgresql-migration)'s bootstrap target, which must +run before this chart. To reproduce compose exactly instead, set +`database.name: registry` and drop the migration step. + +## Probes + +These follow Sunbird's own chart rather than the compose healthcheck. Compose +curls the WildFly management port (9990); this chart does not expose 9990, +because nothing in-cluster needs it and the admin console is on 8080 under +`/auth/admin`. Readiness instead checks that Keycloak is really serving `/auth`. + +First start imports the realm and migrates the schema, so `startupProbe` carries +the slow path (up to 5 minutes by default) and liveness stays tight. + +## Image tag + +Compose uses `latest`. A chart must not deploy a moving tag, so this defaults to +`v1.0.0` — the version Sunbird's own v2 deployment pairs with +`sunbird-rc-core:v2.0.0`. Set `image.digest` to pin exactly. + +## Ingress and the issuer URL + +`proxyAddressForwarding` is on, matching compose, so Keycloak will honour +`X-Forwarded-Host` / `X-Forwarded-Proto`. If you enable ingress: + +- The controller must actually set those headers. +- The registry's `keycloak.url` must be whichever URL Keycloak puts in the `iss` + claim. A mismatch rejects every authenticated request, and the failure reads + like a permissions problem rather than a configuration one. + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema, and +[`examples/keycloak.dev.yaml`](./examples/keycloak.dev.yaml) for a per-environment +file. + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/keycloak/ci/lint-values.yaml b/charts/keycloak/ci/lint-values.yaml new file mode 100644 index 0000000..85da7a6 --- /dev/null +++ b/charts/keycloak/ci/lint-values.yaml @@ -0,0 +1,12 @@ +# Minimum values that let `helm template` run in CI. +# +# This chart deliberately fails the render when the database and password +# Secrets are unset, so bare defaults cannot be templated. These are dummy +# references, not real configuration - see examples/ for that. +database: + host: registry-db-rw + passwordSecret: + name: registry-db-app +admin: + passwordSecret: + name: keycloak-admin diff --git a/charts/keycloak/examples/keycloak.dev.yaml b/charts/keycloak/examples/keycloak.dev.yaml new file mode 100644 index 0000000..fbc9cc3 --- /dev/null +++ b/charts/keycloak/examples/keycloak.dev.yaml @@ -0,0 +1,77 @@ +# Example: Keycloak for the registry, dev environment. +# +# helm install keycloak charts/keycloak \ +# -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml +# +# Install order: postgresql-cnpg -> THIS -> regenerate admin-api secret -> +# registry. See charts/registry/README.md. + +# No fullnameOverride needed: the chart is named "keycloak", so a release named +# "keycloak" already produces Service/keycloak. Set it only if you install under +# a different release name and still want the service to be reachable as +# "keycloak" - the registry validates the token issuer against this host. + +# Keycloak gets its OWN database and its OWN role, both created by the +# postgresql-cnpg chart (`databases` and `managed.roles`). This is where the +# cluster deployment deliberately improves on compose, which shares one +# `registry` database between both services as the postgres superuser: Keycloak's +# realm tables have no reason to sit beside registry records, and nothing here +# needs superuser rights. +# +# To reproduce compose exactly instead, set name: registry, user: postgres, and +# point passwordSecret at registry-db-superuser. +database: + host: registry-db-rw + port: 5432 + name: keycloak + user: keycloak + # The same Secret the managed role is reconciled against, so the password the + # role has and the password Keycloak sends cannot drift. + passwordSecret: + name: keycloak-db + key: password + +admin: + username: admin + # Create by hand until ESO is available: + # kubectl -n oan-registry create secret generic keycloak-admin \ + # --from-literal=keycloakAdminPassword='' + passwordSecret: + name: keycloak-admin + key: keycloakAdminPassword + +# Ships the realm the registry expects. Its admin-api client secret is masked +# and must be regenerated in the console after first start. +# Uses the realm shipped with the chart (files/realm-export.json), so nothing +# needs preparing before install. +realmImport: + enabled: true + +# Dev: reach it with `kubectl port-forward` rather than an ingress. Enabling +# ingress means the controller must set X-Forwarded-Host / X-Forwarded-Proto, +# and the registry's keycloak.url must then be the external URL. +ingress: + enabled: false + +resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +commonLabels: + oan.in/environment: dev + +# Blocks startup until the database accepts connections, standing in for +# compose's `depends_on: condition: service_healthy`. Host and port are taken +# from `database` above. +waitFor: + enabled: true + database: true + +# `helm test keycloak -n oan-registry` checks /auth and the imported realm. +tests: + enabled: true + realm: sunbird-rc diff --git a/charts/keycloak/examples/keycloak.prod.yaml b/charts/keycloak/examples/keycloak.prod.yaml new file mode 100644 index 0000000..e64d40b --- /dev/null +++ b/charts/keycloak/examples/keycloak.prod.yaml @@ -0,0 +1,97 @@ +# Example: Keycloak for the registry, production. +# +# helm install keycloak charts/keycloak \ +# -n oan-registry -f charts/keycloak/examples/keycloak.prod.yaml + +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-keycloak + tag: "v1.0.0" + # digest: "sha256:..." # pin this before going live + +# NOTE: this is the legacy WildFly Keycloak distribution. Running two replicas +# needs its Infinispan cache cluster configured (JGroups discovery, session +# replication) or the two will not share sessions - a token issued by one is not +# recognised by the other. That is not configured by this chart, so production +# stays at one replica until either that is set up or the deployment moves to a +# current Keycloak with the Keycloak Operator. +# +# The practical consequence: a Keycloak restart briefly interrupts token issuing. +# The registry keeps serving requests whose tokens are already valid. +replicaCount: 1 + +database: + host: registry-db-rw + port: 5432 + name: keycloak + # Production uses a dedicated role owning only the keycloak database, not the + # superuser. Requires that role and database to exist - see + # postgresql-migration. + user: keycloak + passwordSecret: + name: keycloak-db + key: password + +admin: + username: admin + passwordSecret: + name: keycloak-admin + key: keycloakAdminPassword + +# The admin password Secret is created out of band, by whatever manages secrets +# in this environment. This chart never renders a password. +# kubectl -n oan-registry create secret generic keycloak-admin \ +# --from-literal=keycloakAdminPassword='' + +# Uses the realm shipped with the chart (files/realm-export.json), so nothing +# needs preparing before install. +realmImport: + enabled: true + +# Reachable from outside only if users need the login page. Keep it internal +# otherwise - the registry reaches Keycloak over cluster DNS. +# +# If you do expose it: proxyAddressForwarding below is already on, the controller +# must set X-Forwarded-Host and X-Forwarded-Proto, and the registry's +# keycloak.url must then be the EXTERNAL URL, because that is what Keycloak will +# put in the token issuer. +ingress: + enabled: false + # className: alb + # annotations: + # alb.ingress.kubernetes.io/scheme: internal + # hosts: + # - host: keycloak.oan.example + # paths: + # - path: /auth + # pathType: Prefix + # tls: + # - hosts: [keycloak.oan.example] + # secretName: keycloak-tls + +proxyAddressForwarding: true + +resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 3Gi + +# Not enabled: with replicaCount 1 a PDB blocks node drains entirely. Revisit +# together with the clustering note above. +podDisruptionBudget: + enabled: false + +podSecurityContext: + enabled: false +securityContext: + enabled: false + +envConfig: + # Keep the JVM heap inside the memory limit above. + JAVA_OPTS_APPEND: "-Xms1024m -Xmx2048m" + +commonLabels: + oan.in/environment: prod diff --git a/charts/keycloak/files/realm-export.json b/charts/keycloak/files/realm-export.json new file mode 100644 index 0000000..4b343c1 --- /dev/null +++ b/charts/keycloak/files/realm-export.json @@ -0,0 +1,2312 @@ +{ + "id": "sunbird-rc", + "realm": "sunbird-rc", + "displayName": "Sunbird Rc Core", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "a772a1cd-7904-4e5c-a864-5041fa69d491", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "42dba8cf-f483-4668-a087-cba46ed86ad2", + "name": "admin", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "id": "5fa4077d-1686-4506-97a6-5bce1bce59dc", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + }, + { + "name": "network_operator", + "description": "Network Operator - onboards and governs Providers", + "composite": false, + "clientRole": false, + "containerId": "sunbird-rc", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "270ecc82-3249-475c-a851-d3ea162059b8", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "0de39ec0-7602-4aa2-b54d-ab12e9bdb76f", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4259031b-736e-49eb-9e70-4a312a48e211", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9887e071-49b0-464b-b6fe-a1c585a709c7", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7d61f967-1dce-482f-96e5-9eff79eb4851", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-events", + "manage-identity-providers", + "manage-clients", + "view-identity-providers", + "manage-authorization", + "view-users", + "manage-users", + "manage-events", + "manage-realm", + "impersonation", + "view-authorization", + "query-clients", + "create-client", + "view-clients", + "query-users", + "query-realms", + "view-realm", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "e93b1761-fb32-46c5-bfa2-4b853c7b5573", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "bc32a137-07a1-40f0-b9fd-a6e64e27f99b", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "4b5abd90-d6a2-4981-a50f-520292496f0b", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "99d2ed5f-00a9-44ed-8b9f-bdd7ba3facb8", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "3fbd2cd5-0698-490e-a52f-ef528d001a62", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "9b8b4f1c-5ed6-49ca-bec3-0a9a4867ad26", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "7e6341ff-a1d8-4400-af94-3a007a06706a", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ad0c87da-9f34-4992-a83a-f6b924f1944d", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "badb0d4d-06da-45e8-a777-ef47f712d3ed", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "f8f48f0f-bd2a-4cb7-9b77-af69b9805c25", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ca0b1e94-6578-4295-abf4-ae99f7df7595", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "ff7230eb-7dae-44a5-8f68-f68747f35589", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "890d054b-86f9-49f5-8dd9-14f62aa956de", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + }, + { + "id": "627f3f4c-58e3-49f3-9989-a05d4d0a8752", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "961a8a92-1598-48ff-adee-1e5fee0df757", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-api": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "2e9bfeff-129e-4072-9617-5847644aac24", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "attributes": {} + } + ], + "account": [ + { + "id": "5694c2d0-6d02-4182-bb09-78f4f5f1ec2d", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0612622f-dae7-48f8-8985-fe7e5ab8acc7", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "eeefbd57-94b8-4d7d-bf2f-075c39ccb746", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "9e9165b9-1170-47ab-802a-aecffefb3ab7", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "a8d0a100-e382-49ba-ac42-48dbf815a2de", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "08772792-146d-4676-ba2d-ce56b0104263", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + }, + { + "id": "0a2e7893-784e-47ef-ba35-4a26901350c0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "01326f76-7838-47fc-ae62-399a75c5ae38", + "attributes": {} + } + ], + "registry-frontend": [] + } + }, + "groups": [], + "defaultRole": { + "id": "8ce3f968-e251-4ea3-a815-c00f9a40815a", + "name": "default-roles-sunbird-rc", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "sunbird-rc" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpSupportedApplications": [ + "FreeOTP", + "Google Authenticator" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "users": [ + { + "id": "3cc9ac60-b67d-4c57-8005-acd4d236b2dc", + "createdTimestamp": 1634296700339, + "username": "service-account-admin-api", + "enabled": true, + "totp": false, + "emailVerified": false, + "serviceAccountClientId": "admin-api", + "disableableCredentialTypes": [], + "requiredActions": [], + "realmRoles": [ + "default-roles-sunbird-rc", + "admin" + ], + "clientRoles": { + "realm-management": [ + "manage-users", + "manage-realm" + ] + }, + "notBefore": 0, + "groups": [] + }, + { + "username": "no-user", + "enabled": true, + "emailVerified": false, + "credentials": [ + { + "type": "password", + "value": "no-user-password", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-sunbird-rc", + "network_operator" + ] + } + ], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account" + ] + } + ] + }, + "clients": [ + { + "id": "01326f76-7838-47fc-ae62-399a75c5ae38", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "f871d6fc-d997-4ac6-99fe-d797955bc9f0", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/sunbird-rc/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/sunbird-rc/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "8ab32c51-9aa0-4e28-80bf-0d6b53151354", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "25962708-6d45-47d9-8935-5db159234aac", + "clientId": "admin-api", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "secret": "**********", + "redirectUris": [ + "*", + "http://localhost:4200/", + "http://localhost:4200/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "84ae9d6c-424f-47f0-9d4d-f2e98fed7339", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + }, + { + "id": "98406938-b8db-4992-8519-917054f6ed0e", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientId", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientId", + "jsonType.label": "String" + } + }, + { + "id": "90d6b17a-5a06-4546-8091-960301f8147e", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b245b10b-606c-417c-bbc0-8f81a7a992a6", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "34e4506c-ea71-4248-a8da-cc2054e9007c", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "961a8a92-1598-48ff-adee-1e5fee0df757", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "69c04ae8-6669-48e7-8234-08986a7f490d", + "clientId": "registry-frontend", + "name": "Registry Frontend", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "saml.encrypt": "false", + "login_theme": "sunbird-rc", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "exclude.session.state.from.auth.response": "false", + "oidc.ciba.grant.enabled": "false", + "saml.artifact.binding": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "saml_force_name_id_format": "false", + "saml.client.signature": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b777b14f-b0e8-4da5-a802-092803319cbe", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/sunbird-rc/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/sunbird-rc/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "7160f35d-97d3-4730-9769-4b03b32e5191", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "b4695333-f842-4ef7-874e-99260e77b9cb", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "fc23d4b8-76c5-4e59-9305-10846b8bcefe", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + }, + { + "id": "08f06ba5-3e60-4a0a-aaf9-f70bfc7ae99e", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e28227ee-cb54-4557-8908-01864f80055f", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "364a632f-b66a-4ca4-8bbe-ec2ce1af9df8", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "42cad815-4de0-4b67-abca-7f7aaf55e589", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "e3302def-d387-465c-a420-7ab01570e94a", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "63a3cb24-b124-428e-ac0f-253eb1fe485d", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "2eb041ca-970a-45fd-a167-2a497579bc8c", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "2c02e9ce-7d86-4a5b-84b8-cf93114ddf26", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "2d39b55e-46c4-4dec-bd83-f081c708f544", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "e869fffd-f801-492d-a6c7-d6c6143817e5", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "daeb863b-4773-4668-98fb-403e93414eb2", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + }, + { + "id": "a98b9f93-ec39-4f3c-acb7-cd92161e3717", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "c59a379a-3934-4e6f-be20-1803b0786d97", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "0a299a91-277c-4f38-95e7-6c520f892b63", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "d51531b3-a8ea-44e2-a48f-69991f9166cc", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "b4b33a89-01db-468e-9a4e-c5ac58304fed", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "d1727e08-fb90-49ce-bb7e-d7a55a50ee64", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "564ae79d-e505-416c-b794-ddd3a3c21fde", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "986d2d9e-0d0d-4317-92b3-a7a8d9bec4de", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "e14ec2e9-0d24-4960-8779-00f769ccc01b", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "8f044609-b615-4522-b9e7-8361cb08b0b3", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "341b838d-ba26-4280-b0af-3e5d3403c938", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "26a17e7e-1a6e-439f-a54a-05a63d1c91fb", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "2489b2c0-5b3a-4404-8428-be4ce653da72", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "8c7e1d96-bf79-42e6-9360-b5e7b8dddc8d", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "01f959d8-123f-4263-ad6e-386e8b4d0e05", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "b47a30d6-3c49-4bbe-b15e-b0eb6cffc0f3", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "2cc8166d-6d77-4a85-9945-bc22b0f550e3", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "1d87a800-cc11-4d85-aa76-8a6d828e2269", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + }, + { + "id": "42cc1538-f83a-4a94-b5a5-d16b80824a02", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "13ff6325-822e-4087-9e74-086de77fe89e", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "e501527e-dec8-4fde-a539-8e77d86b5081", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "b4b519d8-070d-4dab-854e-d6e3b2b36205", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + } + ], + "defaultDefaultClientScopes": [ + "role_list", + "profile", + "email", + "roles", + "web-origins" + ], + "defaultOptionalClientScopes": [ + "offline_access", + "address", + "phone", + "microprofile-jwt" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "ed42958b-6e78-42a9-9f40-2e40bd6c8dd0", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-attribute-mapper", + "saml-user-property-mapper", + "oidc-usermodel-property-mapper", + "oidc-address-mapper", + "saml-role-list-mapper", + "oidc-full-name-mapper" + ] + } + }, + { + "id": "572219a7-3053-4940-87c5-ad94a6fb6dd3", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "8704a420-bf90-4e12-9e33-d21f39a2385b", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "8cf98455-916b-487a-8322-3f5d283400c2", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "0b43488d-108b-41f5-ab6d-56a4ac8ff63c", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-role-list-mapper", + "oidc-address-mapper", + "saml-user-attribute-mapper", + "oidc-usermodel-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "6f0ebf9b-900a-4ca9-8fea-90719f218689", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "b5a486b3-abf9-49a1-8dc6-dc5e20776681", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "be43420e-8ffc-4f53-b745-f2f0cd88f000", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "f749bd77-72f2-4dc4-a65e-dd89b255f12f", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "a541cbb1-8a27-4061-a389-9f24ba1c2eb1", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "18504bf7-63f1-4848-b565-6348fa6b0048", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "497d8386-9a74-4b7b-a4e6-78bbbbb5d795", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "d964973c-2106-4db3-a814-f7a34ae7a1ce", + "alias": "Authentication Options", + "description": "Authentication options.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ff1b250-85b1-4709-8719-3eabcb34493f", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "43f683be-52e7-43cd-aa9e-6318b8079ad0", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "5195e46e-b2c3-49e3-8987-db8b19c45fc5", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "aca5480d-842c-4fa9-aff1-b8af8d51d82a", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Account verification options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "66a00ea9-7ec1-4450-905a-14b7f3f8e4bf", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "a24445a4-1988-4b5b-bde6-fa36dbd07e03", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "31c2bc3b-6eb1-4c4f-8464-3528f7445ef7", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4ccc8da9-0e1e-4f30-99c0-e2139f671a80", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "flowAlias": "forms", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "fa1183d1-af7a-40dd-ba85-d7d37867639c", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "f5b5e49b-7cc9-4011-b9fa-60f0ef65e735", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "5cf727d2-e25a-4c88-a55b-4eea9134adb1", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "3ec4f009-2f16-464b-8feb-a0bdc0dad195", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "User creation or linking", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "a695a5e0-326f-4658-8518-a1769d97ad5f", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "4611e0a9-a6a4-4e32-8500-e68877b464b1", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Authentication Options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "55be727b-a17b-40c5-a5c3-c2d72c7f54cb", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "flowAlias": "registration form", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "00ec3b72-3abb-4db3-ad2f-595bc2f7e086", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "82e4f3a3-744b-4d8a-8785-6eabaf9e05c9", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "0c0312b2-db7c-433c-ab15-20b18bfb5f4a", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "ee0faa63-999c-42e6-8189-c22a5cc14dc5", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "506eed8f-88c9-4978-b13a-886f1efc45c0", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "oauth2DevicePollingInterval": "5", + "clientOfflineSessionMaxLifespan": "0", + "clientSessionIdleTimeout": "0", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5" + }, + "keycloakVersion": "14.0.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} \ No newline at end of file diff --git a/charts/keycloak/templates/NOTES.txt b/charts/keycloak/templates/NOTES.txt new file mode 100644 index 0000000..f32bde6 --- /dev/null +++ b/charts/keycloak/templates/NOTES.txt @@ -0,0 +1,36 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "keycloak.fullname" . }} ({{ .Values.replicaCount }} replica(s)) + - Service/{{ include "keycloak.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "keycloak.envConfigMapName" . }} +{{- if include "keycloak.renderRealmConfigMap" . }} + - ConfigMap/{{ include "keycloak.realmConfigMapName" . }} (realm import) +{{- end }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "keycloak.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "keycloak.fullname" . }} +{{- end }} + +Database: {{ .Values.database.user }}@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }} + +Give the registry chart this URL: + keycloak.url: {{ include "keycloak.url" . }} + +{{ if .Values.realmImport.enabled }} +NEXT STEP - the registry cannot authenticate until you do this: + + The realm export ships with a masked admin-api client secret, so it is not a + working credential. Once Keycloak is up: + + 1. kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "keycloak.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + 2. Open http://localhost:{{ .Values.service.port }}/auth/admin and sign in as "{{ .Values.admin.username }}". + 3. Realm "sunbird-rc" -> Clients -> admin-api -> Credentials -> Regenerate Secret. + 4. Store that value, and set it as the registry's admin client secret. +{{- end }} + +Watch the rollout (first start imports the realm and migrates the schema, so +give it a few minutes): + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "keycloak.fullname" . }} diff --git a/charts/keycloak/templates/_helpers.tpl b/charts/keycloak/templates/_helpers.tpl new file mode 100644 index 0000000..8557b64 --- /dev/null +++ b/charts/keycloak/templates/_helpers.tpl @@ -0,0 +1,159 @@ +{{/* +# ============================================================================ +# KEYCLOAK (SUNBIRD RC) CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the realm-import +# and legacy-Keycloak database wiring this chart needs. +# ============================================================================ +*/}} + +{{- define "keycloak.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "keycloak.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "keycloak.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "keycloak.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "keycloak.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "keycloak.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "keycloak.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +The base URL other services use to reach this Keycloak, including the /auth +context path the legacy distribution serves under. The registry's issuer check +compares against exactly this, so it must match what the registry is given. +*/}} +{{- define "keycloak.url" -}} +{{- printf "http://%s.%s.svc.cluster.local:%v/auth" (include "keycloak.fullname" .) .Release.Namespace .Values.service.port -}} +{{- end }} + +{{/* +Name of the ConfigMap holding the realm export. + +The chart renders its own by default. An existingConfigMap wins, for the case +where the realm is managed outside the chart entirely. +*/}} +{{- define "keycloak.realmConfigMapName" -}} +{{- if .Values.realmImport.existingConfigMap -}} +{{- .Values.realmImport.existingConfigMap -}} +{{- else -}} +{{- printf "%s-realm" (include "keycloak.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Emits "true" when this chart renders the realm ConfigMap itself, i.e. whenever +the realm is not being supplied by an out-of-band ConfigMap. +*/}} +{{- define "keycloak.renderRealmConfigMap" -}} +{{- if and .Values.realmImport.enabled (not .Values.realmImport.existingConfigMap) -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +The realm JSON: an inline override if given, otherwise the file shipped in the +chart under files/. +*/}} +{{- define "keycloak.realmJson" -}} +{{- if .Values.realmImport.realmJson -}} +{{- .Values.realmImport.realmJson -}} +{{- else -}} +{{- $json := .Files.Get .Values.realmImport.file -}} +{{- if not $json -}} +{{- fail (printf "%s: realmImport is enabled but %q is not present in the chart, and neither realmImport.realmJson nor realmImport.existingConfigMap is set." .Chart.Name .Values.realmImport.file) -}} +{{- end -}} +{{- $json -}} +{{- end -}} +{{- end }} + +{{/* +Path the realm file is mounted at, which is what KEYCLOAK_IMPORT points to. +*/}} +{{- define "keycloak.realmImportPath" -}} +{{- printf "%s/%s" (.Values.realmImport.mountPath | trimSuffix "/") .Values.realmImport.fileName -}} +{{- end }} + +{{/* +Environment for the legacy (WildFly) Keycloak distribution. This image predates +Keycloak's KC_* variables, so it takes DB_VENDOR/DB_ADDR and friends. Values +mirror the compose stack. +*/}} +{{- define "keycloak.env" -}} +{{- $db := .Values.database }} +{{- if not $db.host }} +{{- fail (printf "%s: database.host is required - point it at the PostgreSQL primary service, e.g. registry-db-rw" .Chart.Name) }} +{{- end }} +{{- if not $db.passwordSecret.name }} +{{- fail (printf "%s: database.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) }} +{{- end }} +{{- if not .Values.admin.passwordSecret.name }} +{{- fail (printf "%s: admin.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) }} +{{- end }} +- name: DB_VENDOR + value: {{ $db.vendor | quote }} +- name: DB_ADDR + value: {{ $db.host | quote }} +- name: DB_PORT + value: {{ $db.port | quote }} +- name: DB_DATABASE + value: {{ $db.name | quote }} +- name: DB_USER + value: {{ $db.user | quote }} +{{- with $db.schema }} +- name: DB_SCHEMA + value: {{ . | quote }} +{{- end }} +- name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $db.passwordSecret.name }} + key: {{ $db.passwordSecret.key }} +- name: KEYCLOAK_USER + value: {{ .Values.admin.username | quote }} +- name: KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.admin.passwordSecret.name }} + key: {{ .Values.admin.passwordSecret.key }} +- name: PROXY_ADDRESS_FORWARDING + value: {{ .Values.proxyAddressForwarding | quote }} +{{- if .Values.realmImport.enabled }} +- name: KEYCLOAK_IMPORT + value: {{ include "keycloak.realmImportPath" . | quote }} +{{- end }} +{{- with (include "oan-common.env" . | trim) }} +{{ . }} +{{- end }} +{{- end }} + +{{/* +Dependency waits, derived from this chart's own settings so there is nothing to +keep in sync: the host comes from database.host, which is the same value the +container connects to. +*/}} +{{- define "keycloak.waitFor" -}} +{{- $tcp := list -}} +{{- if .Values.waitFor.database }} +{{- $tcp = append $tcp (dict "name" "database" "host" .Values.database.host "port" .Values.database.port) -}} +{{- end -}} +{{- $tcp = concat $tcp (.Values.waitFor.extraTcp | default list) -}} +{{- include "oan-common.waitFor" (dict "ctx" . "tcp" $tcp "http" (.Values.waitFor.extraHttp | default list)) -}} +{{- end }} diff --git a/charts/keycloak/templates/configmap-realm.yaml b/charts/keycloak/templates/configmap-realm.yaml new file mode 100644 index 0000000..6890099 --- /dev/null +++ b/charts/keycloak/templates/configmap-realm.yaml @@ -0,0 +1,27 @@ +{{- if include "keycloak.renderRealmConfigMap" . }} +{{/* +The realm the registry expects: realm `sunbird-rc`, the admin-api and +registry-frontend clients, and the roles the registry checks. Mounted at +realmImport.mountPath and imported on startup via KEYCLOAK_IMPORT. + +The exported admin-api client secret is masked ("**********"), not a working +credential - it has to be regenerated in the Keycloak console after the first +import. See this chart's README. + +Not rendered when realmImport.existingConfigMap points at a ConfigMap managed +outside the chart. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "keycloak.realmConfigMapName" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{ .Values.realmImport.fileName }}: |- + {{- include "keycloak.realmJson" . | nindent 4 }} +{{- end }} diff --git a/charts/keycloak/templates/configmap.yaml b/charts/keycloak/templates/configmap.yaml new file mode 100644 index 0000000..ed67084 --- /dev/null +++ b/charts/keycloak/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "keycloak.envConfigMapName" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with (include "oan-common.envConfigMapData" . | trim) }} +data: + {{- . | nindent 2 }} +{{- else }} +data: {} +{{- end }} diff --git a/charts/keycloak/templates/deployment.yaml b/charts/keycloak/templates/deployment.yaml new file mode 100644 index 0000000..bca671f --- /dev/null +++ b/charts/keycloak/templates/deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + # The legacy distribution is not clustered here, and the realm import runs on + # startup, so replicas are rolled one at a time with the old pod gone first. + strategy: + type: Recreate + selector: + matchLabels: + {{- include "keycloak.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "keycloak.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + {{- if include "keycloak.renderRealmConfigMap" . }} + {{/* + Rolls the pod when the realm changes. Only possible while the chart owns + the realm content - with realmImport.existingConfigMap, Helm cannot read + cluster state to checksum it, so restart Keycloak yourself after editing + that ConfigMap. + */}} + checksum/realm: {{ include "keycloak.realmJson" . | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "keycloak.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + {{- $init := include "keycloak.waitFor" . | trim }} + {{- $extraInit := "" }} + {{- with .Values.extraInitContainers }} + {{- $extraInit = toYaml . | trim }} + {{- end }} + {{- if or $init $extraInit }} + initContainers: + {{- with $init }} + {{- . | nindent 8 }} + {{- end }} + {{- with $extraInit }} + {{- . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "keycloak.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- include "keycloak.env" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "keycloak.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + {{- if or .Values.realmImport.enabled .Values.extraVolumeMounts }} + volumeMounts: + {{- if .Values.realmImport.enabled }} + - name: realm + mountPath: {{ .Values.realmImport.mountPath }} + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.realmImport.enabled .Values.extraVolumes }} + volumes: + {{- if .Values.realmImport.enabled }} + - name: realm + configMap: + name: {{ include "keycloak.realmConfigMapName" . }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/keycloak/templates/ingress.yaml b/charts/keycloak/templates/ingress.yaml new file mode 100644 index 0000000..1e99eb8 --- /dev/null +++ b/charts/keycloak/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "keycloak.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/keycloak/templates/poddisruptionbudget.yaml b/charts/keycloak/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..f210bd2 --- /dev/null +++ b/charts/keycloak/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Caps how many pods can be voluntarily disrupted at once - node drains, cluster +upgrades, autoscaler scale-down. + +With replicaCount: 1 a PDB of minAvailable: 1 blocks drains entirely, because +evicting the only pod would breach it. That is why this is disabled by default +and should be enabled together with replicaCount >= 2. +*/}} +{{- if and (eq (int .Values.replicaCount) 1) (not .Values.podDisruptionBudget.allowSingleReplica) }} +{{- fail (printf "%s: podDisruptionBudget is enabled with replicaCount 1, which blocks node drains entirely. Raise replicaCount to 2+, or set podDisruptionBudget.allowSingleReplica=true if you accept that." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "keycloak.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/keycloak/templates/service.yaml b/charts/keycloak/templates/service.yaml new file mode 100644 index 0000000..1a824dd --- /dev/null +++ b/charts/keycloak/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "keycloak.fullname" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "keycloak.selectorLabels" . | nindent 4 }} diff --git a/charts/keycloak/templates/serviceaccount.yaml b/charts/keycloak/templates/serviceaccount.yaml new file mode 100644 index 0000000..83b8800 --- /dev/null +++ b/charts/keycloak/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "keycloak.serviceAccountName" . }} + labels: + {{- include "keycloak.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/keycloak/templates/tests/test-connection.yaml b/charts/keycloak/templates/tests/test-connection.yaml new file mode 100644 index 0000000..5c31958 --- /dev/null +++ b/charts/keycloak/templates/tests/test-connection.yaml @@ -0,0 +1,39 @@ +{{- if .Values.tests.enabled }} +{{/* +`helm test` check: Keycloak is serving its realm endpoint. + +This is the check that matters for the stack - the registry validates tokens +against this exact URL, so if it does not answer, the registry cannot +authenticate no matter how healthy the pod looks. +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "keycloak.fullname" . }}-test-connection + labels: + {{- include "keycloak.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: test + image: {{ printf "%s/%s:%s" .Values.tests.image.registry .Values.tests.image.repository .Values.tests.image.tag | quote }} + command: + - /bin/sh + - -c + - | + set -e + base="http://{{ include "keycloak.fullname" . }}:{{ .Values.service.port }}/auth" + echo "GET $base/" + wget -q -O /dev/null --timeout=10 "$base/" + echo "OK: /auth is serving" + {{- if .Values.realmImport.enabled }} + echo "GET $base/realms/{{ .Values.tests.realm }}" + wget -q -O /dev/null --timeout=10 "$base/realms/{{ .Values.tests.realm }}" + echo "OK: realm {{ .Values.tests.realm }} is imported and serving" + {{- end }} + resources: + {{- toYaml .Values.tests.resources | nindent 8 }} +{{- end }} diff --git a/charts/keycloak/values.yaml b/charts/keycloak/values.yaml new file mode 100644 index 0000000..0391081 --- /dev/null +++ b/charts/keycloak/values.yaml @@ -0,0 +1,313 @@ +# ============================================================================ +# keycloak - default values +# +# Keycloak for the OAN registry, on the Sunbird RC Keycloak image. This is the +# legacy (WildFly/JBoss) Keycloak distribution: it serves under /auth and takes +# DB_VENDOR-style environment variables, not the modern KC_* ones. +# +# Defaults mirror the verified compose stack (registry/docker-compose.yml) so a +# cluster deploy reproduces what was tested locally. Where this chart deviates, +# it says so and why. +# +# Minimum you must set: +# - database.host (e.g. the CNPG primary, registry-db-rw) +# - database.passwordSecret.name +# - admin.passwordSecret.name +# ============================================================================ + +replicaCount: 1 + +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-keycloak + # Compose uses `latest`. A chart must not deploy a moving tag, so this pins + # the version Sunbird's own v2 deployment pairs with sunbird-rc-core v2.0.0. + tag: "v1.0.0" + # Digest pin; wins over tag. Preferred for anything past dev. + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +# Only needed when the release name differs from "keycloak". The resulting name +# is the in-cluster DNS the registry connects to, and the registry validates the +# token issuer against exactly this host - so keep it stable once anything +# depends on it. +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: true + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + annotations: {} +# Compose also publishes 9990, the WildFly management port. That exists only for +# the compose healthcheck; nothing in-cluster needs it, and the Keycloak admin +# console is on 8080 under /auth/admin. Not exposed here. + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: keycloak.local + paths: + # Legacy Keycloak serves everything under /auth + - path: /auth + pathType: Prefix + tls: [] + +# --------------------------------------------------------------------------- +# Database +# +# Defaults match compose: Keycloak shares the registry's database and connects +# as the same user. Sunbird's own Helm charts instead give Keycloak its own +# database ("keycloak") on the same host, which is the better arrangement - +# Keycloak's realm tables have no reason to sit beside registry records, and it +# frees the registry to connect as a non-superuser owner. To separate them: +# +# database: +# name: keycloak +# user: keycloak +# +# and provision that database and role in the cluster. +# --------------------------------------------------------------------------- +database: + vendor: postgres + # PostgreSQL primary service, e.g. "registry-db-rw" + host: "" + port: 5432 + name: registry + user: postgres + schema: "" + # Secret holding the database password. Required. + passwordSecret: + name: "" + key: password + +# --------------------------------------------------------------------------- +# Keycloak admin user (the console login) +# --------------------------------------------------------------------------- +admin: + username: admin + passwordSecret: + name: "" + key: keycloakAdminPassword + +# --------------------------------------------------------------------------- +# Realm import +# +# The chart ships the realm the registry expects: realm `sunbird-rc`, the +# admin-api and registry-frontend clients, and the roles the registry checks. +# The legacy image imports it on startup when the realm is not already present. +# +# IMPORTANT: the exported admin-api client secret is masked ("**********"), not +# a working credential. After the first import, regenerate it in the console +# (realm -> Clients -> admin-api -> Credentials -> Regenerate Secret) and hand +# that value to the registry chart. First install is therefore two-phase. +# --------------------------------------------------------------------------- +realmImport: + enabled: true + + # The realm shipped with this chart, rendered into a ConfigMap and mounted for + # KEYCLOAK_IMPORT. Copied verbatim from the compose stack's + # registry/imports/realm-export.json, so the cluster imports the same realm + # that was verified locally. + # + # Keeping it in the chart means `helm install` needs nothing prepared + # beforehand, and a realm change rolls the pod via the checksum annotation. + # The trade-off is that this is a second copy of that file: if the compose one + # changes, update this one too. + file: files/realm-export.json + + # Key in the ConfigMap, which becomes the filename on disk and the value of + # KEYCLOAK_IMPORT. + fileName: realm-export.json + + # Where the legacy image looks for imports. + mountPath: /opt/jboss/keycloak/imports + + # Use a ConfigMap managed outside this chart instead. When set, the chart + # renders no realm ConfigMap and mounts this name - so it must already exist. + existingConfigMap: "" + + # Inline realm JSON, overriding the shipped file. Useful for a test with a + # deliberately minimal realm. + realmJson: "" + +# Lets a caller behind an ingress obtain a token whose `iss` matches what the +# registry validates, via X-Forwarded-Host / X-Forwarded-Proto. Matches compose. +# If you enable ingress, the controller must actually set those headers. +proxyAddressForwarding: true + +# WildFly-based Keycloak is memory-hungry; Sunbird requests 500m/2G with no +# ceiling. Limits are mandatory in this repo, so it gets one. +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +# --------------------------------------------------------------------------- +# Probes +# +# These follow Sunbird's own Helm chart rather than the compose healthcheck: +# compose curls the WildFly management port (9990), which this chart does not +# expose. Readiness instead checks that Keycloak is really serving /auth. +# +# First start imports the realm and runs schema migrations, so startupProbe +# carries the slow path and liveness stays tight. +# --------------------------------------------------------------------------- +startupProbe: + enabled: true + httpGet: + path: /auth/ + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 30 + +livenessProbe: + enabled: true + tcpSocket: + port: http + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 10 + +readinessProbe: + enabled: true + httpGet: + path: /auth/ + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 10 + +# This image runs as the jboss user and writes into its own directories, so +# readOnlyRootFilesystem needs extra volumes to work. Off by default. +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# --------------------------------------------------------------------------- +# Dependency waiting +# +# Kubernetes has no equivalent of compose's `depends_on: condition: +# service_healthy`. Without these init containers the pod starts before its +# dependencies are up, fails, and crashloops with backoff - which recovers on +# its own but makes a first install look broken. +# --------------------------------------------------------------------------- +waitFor: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + timeoutSeconds: 300 + intervalSeconds: 3 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Wait for the database. Host and port come from `database` above, so there is + # nothing to keep in sync. + database: true + # Extra checks, if this Keycloak depends on anything else. + extraTcp: [] + extraHttp: [] + +# Extra init containers, appended after the waitFor ones. Rendered verbatim. +extraInitContainers: [] + +# Extra volumes and mounts, rendered verbatim. For certificates, extra config, +# or a writable scratch directory when readOnlyRootFilesystem is enabled. +extraVolumes: [] +extraVolumeMounts: [] + +# --------------------------------------------------------------------------- +# PodDisruptionBudget +# +# Caps voluntary disruptions - node drains, cluster upgrades, autoscaler +# scale-down. Disabled by default because with a single replica a PDB of +# minAvailable: 1 blocks drains entirely; the render fails if you enable it +# anyway without acknowledging that. +# --------------------------------------------------------------------------- +podDisruptionBudget: + enabled: false + minAvailable: 1 + # Set one or the other, not both. + maxUnavailable: "" + # Enable a PDB with a single replica anyway, accepting that node drains block. + allowSingleReplica: false + +# --------------------------------------------------------------------------- +# `helm test` checks +# +# helm test -n +# +# Runs after install to confirm the service actually answers, which is the +# difference between "the pod is running" and "the deployment works". +# --------------------------------------------------------------------------- +tests: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Realm the test expects to find. Only used when realmImport.enabled. + realm: sunbird-rc + + +# Extra non-secret configuration, exposed as a ConfigMap and injected with +# envFrom. The database and admin settings above render as explicit env vars, +# so they do not belong here. +envConfig: {} + # JAVA_OPTS_APPEND: "-Xms512m -Xmx1024m" + +# Extra env vars from specific Secret keys, beyond the two wired above. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} From a67a11050ac735fb3532805402301364db8e38c1 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:35:20 +0530 Subject: [PATCH 53/81] feat: add registry and discovery service charts [#53] registry runs Sunbird RC with its Participant schema mounted from a ConfigMap, and connects as the owner of the registry database rather than as postgres. discovery renders the Beckn discovery service, taking its whole DSN from the CNPG-generated Secret so no password is assembled or escaped in the chart. It enables readOnlyRootFilesystem, which the other service charts cannot yet. The Beckn spec is fetched by URL with a cache fallback, and the render fails when neither a URL nor an existing ConfigMap is set, since the service refuses to boot without the document. Both charts fail the render on an unset credential rather than defaulting one. --- charts/discovery/.helmignore | 13 + charts/discovery/CHANGELOG.md | 82 ++++ charts/discovery/Chart.yaml | 29 ++ charts/discovery/README.md | 320 ++++++++++++++ charts/discovery/ci/lint-values.yaml | 20 + charts/discovery/ci/url-secret-values.yaml | 53 +++ charts/discovery/examples/discovery.dev.yaml | 85 ++++ charts/discovery/examples/discovery.prod.yaml | 122 +++++ charts/discovery/templates/NOTES.txt | 59 +++ charts/discovery/templates/_helpers.tpl | 178 ++++++++ charts/discovery/templates/configmap.yaml | 16 + charts/discovery/templates/deployment.yaml | 111 +++++ charts/discovery/templates/hpa.yaml | 54 +++ charts/discovery/templates/ingress.yaml | 41 ++ .../templates/poddisruptionbudget.yaml | 33 ++ charts/discovery/templates/service.yaml | 19 + .../discovery/templates/serviceaccount.yaml | 13 + .../templates/tests/test-connection.yaml | 40 ++ charts/discovery/values.yaml | 416 ++++++++++++++++++ charts/registry/.helmignore | 13 + charts/registry/CHANGELOG.md | 74 ++++ charts/registry/Chart.yaml | 26 ++ charts/registry/README.md | 244 ++++++++++ charts/registry/ci/lint-values.yaml | 18 + charts/registry/examples/registry.dev.yaml | 84 ++++ charts/registry/examples/registry.prod.yaml | 144 ++++++ .../registry/files/schemas/Participant.json | 170 +++++++ charts/registry/templates/NOTES.txt | 35 ++ charts/registry/templates/_helpers.tpl | 181 ++++++++ .../registry/templates/configmap-schemas.yaml | 18 + charts/registry/templates/configmap.yaml | 16 + charts/registry/templates/deployment.yaml | 108 +++++ charts/registry/templates/hpa.yaml | 51 +++ charts/registry/templates/ingress.yaml | 41 ++ .../templates/poddisruptionbudget.yaml | 33 ++ charts/registry/templates/service.yaml | 19 + charts/registry/templates/serviceaccount.yaml | 13 + .../templates/tests/test-connection.yaml | 36 ++ charts/registry/values.yaml | 353 +++++++++++++++ 39 files changed, 3381 insertions(+) create mode 100644 charts/discovery/.helmignore create mode 100644 charts/discovery/CHANGELOG.md create mode 100644 charts/discovery/Chart.yaml create mode 100644 charts/discovery/README.md create mode 100644 charts/discovery/ci/lint-values.yaml create mode 100644 charts/discovery/ci/url-secret-values.yaml create mode 100644 charts/discovery/examples/discovery.dev.yaml create mode 100644 charts/discovery/examples/discovery.prod.yaml create mode 100644 charts/discovery/templates/NOTES.txt create mode 100644 charts/discovery/templates/_helpers.tpl create mode 100644 charts/discovery/templates/configmap.yaml create mode 100644 charts/discovery/templates/deployment.yaml create mode 100644 charts/discovery/templates/hpa.yaml create mode 100644 charts/discovery/templates/ingress.yaml create mode 100644 charts/discovery/templates/poddisruptionbudget.yaml create mode 100644 charts/discovery/templates/service.yaml create mode 100644 charts/discovery/templates/serviceaccount.yaml create mode 100644 charts/discovery/templates/tests/test-connection.yaml create mode 100644 charts/discovery/values.yaml create mode 100644 charts/registry/.helmignore create mode 100644 charts/registry/CHANGELOG.md create mode 100644 charts/registry/Chart.yaml create mode 100644 charts/registry/README.md create mode 100644 charts/registry/ci/lint-values.yaml create mode 100644 charts/registry/examples/registry.dev.yaml create mode 100644 charts/registry/examples/registry.prod.yaml create mode 100644 charts/registry/files/schemas/Participant.json create mode 100644 charts/registry/templates/NOTES.txt create mode 100644 charts/registry/templates/_helpers.tpl create mode 100644 charts/registry/templates/configmap-schemas.yaml create mode 100644 charts/registry/templates/configmap.yaml create mode 100644 charts/registry/templates/deployment.yaml create mode 100644 charts/registry/templates/hpa.yaml create mode 100644 charts/registry/templates/ingress.yaml create mode 100644 charts/registry/templates/poddisruptionbudget.yaml create mode 100644 charts/registry/templates/service.yaml create mode 100644 charts/registry/templates/serviceaccount.yaml create mode 100644 charts/registry/templates/tests/test-connection.yaml create mode 100644 charts/registry/values.yaml diff --git a/charts/discovery/.helmignore b/charts/discovery/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/discovery/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/discovery/CHANGELOG.md b/charts/discovery/CHANGELOG.md new file mode 100644 index 0000000..873dda1 --- /dev/null +++ b/charts/discovery/CHANGELOG.md @@ -0,0 +1,82 @@ +# Changelog + +All notable changes to the `discovery` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-09-01 + +Initial release. Configuration ported from the verified +`discovery-service/docker-compose.yml` stack, built on the `oan-common` library. + +### Added +- Deployment, Service, env ConfigMap, ServiceAccount, and optional Ingress, + HorizontalPodAutoscaler and PodDisruptionBudget. +- Two ways to supply `DATABASE_URL`, exactly one of which must be set: + `database.urlSecret` for the whole DSN from one Secret key — which is what + CloudNativePG's generated `-app` Secret already provides under `uri` + — or `database.host/port/name/user/sslMode` plus `passwordSecret`, from which + the chart assembles a DSN with the password injected through Kubernetes' + `$(VAR)` expansion so it stays in the Secret and out of the rendered manifest. +- Every other environment variable derived from structured values rather than + restated, so `SERVER_PORT` cannot drift from the container port and + `DATABASE_URL` cannot drift from `database.*`. +- Beckn specification wiring: `becknSpec.url` for the fetch path, + `becknSpec.existingConfigMap` for the air-gapped one, and a volume mounted at + the cache path either way — `/app` is not writable by uid 65532, so with + `readOnlyRootFilesystem` on the fetch would have nowhere to cache. +- Split probes: liveness and startup on `/healthz`, readiness on `/readyz`. + Liveness on `/readyz` would restart every pod during a database blip, turning + a recoverable outage into a crashloop across the Deployment. +- Render-time validation of `app.networkId`, the database DSN, the Beckn spec + source, and the endpoint that a non-default `embeddings.provider` or + `otel.exporter` requires. Each names the value and why the boot needs it. +- `helm test` check against `/readyz` rather than `/healthz`: `/healthz` is 200 + with the database down and would pass on a deployment that cannot serve a + single discover. +- `extraVolumes` / `extraVolumeMounts` / `extraInitContainers` escape hatches. +- No init container waiting on PostgreSQL, unlike the `registry` chart. The + service fails fast when it cannot connect and Kubernetes' restart backoff is + already the retry, so a wait container buys a tidier first install at the cost + of a second thing to configure and keep pointed at the right host. +- Per-environment example values for dev and production. Production mounts the + Beckn document from a ConfigMap rather than fetching it, sets a + PodDisruptionBudget and pod anti-affinity, and leaves `image.repository` as a + documented TODO. + +### Changed from the compose stack +- `DATABASE_AUTO_MIGRATE` defaults to `false`. The compose file states the + intent directly: migrating is a step someone decides to take. The README + documents the scale-to-one-replica upgrade that does it. +- The rate limiter is left unset, so the service's own 20 rps / 40 burst + defaults apply. Compose's 100000/100000 is a local-stack ceiling, not a + deployment value. +- `envConfig` is empty. The image already carries the reviewed + `config/common.yaml` defaults and the environment layer sits above it; + restating them here would be a second copy of decisions this chart did not + make. +- Both security contexts are on by default, unlike the other OAN service + charts. The image is distroless/static as uid 65532, so + `readOnlyRootFilesystem` costs nothing once the spec cache is a volume. +- `automountServiceAccountToken` is false: the service calls no Kubernetes API. + +### Known prerequisite +- PostgreSQL must have **pgvector**, and must create the `vector` extension at + bootstrap. `postgresql-cnpg` 0.2.1 ships + `examples/discovery-db.dev.yaml`, which does both: it pins + `dhi.io/pgvector:0.8-pg16` and creates the extension in + `postInitApplicationSQL`. + + The bootstrap half is not optional. `vector` is not a trusted extension, so + the database owner this service connects as cannot create it - only a + superuser can, and `enableSuperuserAccess` is false. Without it the first + statement of the migration fails and the pod crashloops on an error that + reads like a credentials problem. `pg_trgm` is trusted and would have + succeeded, which is what makes the failure look selective and confusing. + + Neither condition is detectable at render time; the README describes both + failure modes. +- No image is published for `discovery-service` — its CI builds and scans one + but pushes nothing — so `image.repository` is empty and the render fails until + an environment supplies a tag. diff --git a/charts/discovery/Chart.yaml b/charts/discovery/Chart.yaml new file mode 100644 index 0000000..8bed016 --- /dev/null +++ b/charts/discovery/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: discovery +description: >- + The OAN Beckn v2.0.0 discover-and-publish service. Holds published catalogs + and answers discover queries over them. PostgreSQL is its only datastore, and + it needs the `vector` and `pg_trgm` extensions. +type: application +version: 0.1.0 +# No image is published for this service yet - CI builds and scans one but +# pushes nothing - so this is the version the chart deploys by default once one +# exists, and `image.repository` is empty until then. See values.yaml. +appVersion: "0.1.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - discovery + - publish + - beckn +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/OpenAgriNet/discovery-service +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/discovery/README.md b/charts/discovery/README.md new file mode 100644 index 0000000..4fa3efa --- /dev/null +++ b/charts/discovery/README.md @@ -0,0 +1,320 @@ +# discovery + +The OAN [discover-and-publish +service](https://github.com/OpenAgriNet/discovery-service) — a Beckn v2.0.0 +`/publish` and `/discover` implementation in Go. PostgreSQL is its only +datastore; there is no spatial extension and no separate search engine. + +Configuration is ported from the verified `discovery-service/docker-compose.yml` +stack, with the two settings that file marks as local-only put back to their +deployment values (see [Differences from compose](#differences-from-compose)). + +## What it renders + +| Resource | Notes | +|---|---| +| Deployment | Probes on `/healthz` and `/readyz`, resources mandatory | +| Service | `ClusterIP` on 8080 | +| ConfigMap (env) | Overrides from `envConfig`. Empty by default — see [Configuration](#configuration) | +| ServiceAccount | `automountServiceAccountToken: false` — the service calls no Kubernetes API | +| HorizontalPodAutoscaler | Optional, off by default | +| PodDisruptionBudget | Optional, off by default | +| Ingress | Optional, off by default | +| Test Pod | `helm test` check against `/readyz` | + +No Secret and no ConfigMap holding the Beckn document. Both are referenced by +name and created outside the chart. + +## Install + +```bash +# 1. The database — pgvector, plus `vector` created at bootstrap. Both matter; see below. +helm install discovery-db charts/postgresql-cnpg -n oan-discovery \ + -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml + +# 2. The service +helm install discovery charts/discovery -n oan-discovery \ + -f charts/discovery/examples/discovery.dev.yaml + +helm test discovery -n oan-discovery +``` + +## The database + +The first migration creates two extensions: + +```sql +CREATE EXTENSION IF NOT EXISTS vector; -- pgvector: the embedding column and its HNSW index +CREATE EXTENSION IF NOT EXISTS pg_trgm; -- contrib: trigram matching for lexical retrieval +``` + +No stock CloudNativePG operand image carries pgvector, so the cluster needs an +image that does. [`postgresql-cnpg`](../postgresql-cnpg) ships a ready example: + +```bash +helm install discovery-db charts/postgresql-cnpg -n oan-discovery \ + -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml +``` + +It pins `dhi.io/pgvector:0.8-pg16` — pgvector 0.8.6 and pg_trgm 1.6 on +PostgreSQL 16, amd64 and arm64. PostgreSQL **16** because that is what this +service is verified against: the design doc names it, `tests/dbtest` pins +`pgvector/pgvector:0.8.0-pg16`, and `EXPLAIN (GENERIC_PLAN)` in that suite is +PG16-and-later only. The schema does apply cleanly to 14, so this is about +staying on the tested major rather than a hard incompatibility. + +Two settings in that example are load-bearing: + +### `vector` is not a trusted extension + +`pg_trgm` is marked `trusted = true`, so a database owner may create it. +`vector` is not: + +``` +ERROR: permission denied to create extension "vector" +HINT: Must be superuser to create this extension. +``` + +This service connects as the bootstrap **owner**, not the superuser, and CNPG's +`enableSuperuserAccess` is false. So without help, the very first statement of +the migration fails — and the pod crashloops on an error that reads like a bad +password rather than a missing grant. + +The example creates both extensions in `bootstrap.postInitApplicationSQL`, which +CNPG runs as superuser inside the freshly created database. That is the one +moment superuser is available without granting it to anything long-lived; after +it, the service's own `IF NOT EXISTS` is a no-op and the rest of the migration +runs as the owner. + +### The image's postgres user is uid 70 + +CNPG defaults to 26 and applies it as the pod's `runAsUser` and `fsGroup`, so +the example sets `postgresUID`/`postgresGID` to 70. Left unset, the data +directory is handed to a user that does not exist in the image. + +### Not an operand image + +`dhi.io/pgvector` is a general-purpose PostgreSQL image. It satisfies CNPG's +documented requirements — `initdb`, `postgres`, `pg_ctl`, `pg_controldata`, +`pg_basebackup` and `du` are all on PATH, and CNPG overrides the image's own +entrypoint — but it is not built from `cloudnative-pg/postgres-containers` and +has not been run under the operator here. **Smoke-test the first cluster to +healthy before pointing this chart at it.** + +It also ships no barman-cloud, which rules out CNPG's in-core backup method. +The Barman Cloud Plugin brings its own sidecar and is unaffected. + +CNPG's generated app Secret carries `username`, `password`, `dbname`, `host`, +`port`, `uri` and `jdbc-uri`. `uri` is the whole DSN, which is why +`database.urlSecret` is the recommended way to wire this chart up. + +### Supplying the DSN + +Exactly one of these, or the render fails: + +```yaml +# Preferred — one Secret key holds the whole DSN. +database: + urlSecret: + name: discovery-db-app + key: uri +``` + +The DSN carries host, port, database and user, so `database.host` and the rest +go unread in this form. Leave them empty rather than setting a second copy that +nothing consumes and nothing keeps true. + +```yaml +# Assembled by the chart, with the password injected through Kubernetes' +# own $(VAR) expansion so it stays in the Secret. +database: + host: discovery-db-rw + port: 5432 + name: discovery + user: discovery + sslMode: prefer + passwordSecret: + name: discovery-db-app + key: password +``` + +The assembled form renders +`postgres://discovery:$(DATABASE_PASSWORD)@discovery-db-rw:5432/discovery?sslmode=prefer`, +and the kubelet substitutes the value of the `DATABASE_PASSWORD` env var — which +comes from the Secret — at container start. The password therefore never appears +in the rendered manifest, in `helm get manifest`, or in anything that logs a +template. + +The one caveat, and the reason `urlSecret` is preferred: that substitution is +textual, so a password containing `@ : / ? # %` must already be percent-encoded +in the Secret or it will silently produce a DSN that parses to the wrong thing. + +### Migrations + +The migrations are compiled into the binary. There is no Job, no sidecar and no +mounted directory: the service applies them on boot when `DATABASE_AUTO_MIGRATE` +is set, and treats "already at the latest version" as success. + +`database.autoMigrate` is **false** by default — migrating is a step someone +decides to take, not something that happens because a pod was rescheduled. + +```bash +helm upgrade discovery -n oan-discovery --reuse-values \ + --set database.autoMigrate=true --set replicaCount=1 +# verify the rollout, then upgrade back with autoMigrate=false +``` + +Scale to one replica first. The migration runs in-process on boot, so several +pods starting at once race to apply the same one. + +## The Beckn specification + +The service loads the Beckn v2.0.0 document **before** it serves and refuses to +start without it. The check is unconditional: turning L1 validation off does not +remove the requirement. + +It tries `becknSpec.url` first and falls back to the on-disk cache, so at least +one of `url` and `existingConfigMap` must be set. The render fails when both are +empty, because that configuration cannot boot. + +| Setting | What happens | +|---|---| +| `url` only | Fetched at boot and cached into an emptyDir. Needs egress, and refetches on every reschedule. The upstream URL tracks `main`, so this pins nothing. | +| `existingConfigMap` only | Mounted read-only at the cache path. No egress needed; reproducible. The boot logs one warning about the fetch it could not do — that warning is accurate and is not a failure. | +| both | Fetch first, ConfigMap as the fallback. | + +```bash +kubectl -n oan-discovery create configmap discovery-beckn-spec \ + --from-file=beckn.yaml=/tests/testdata/beckn-v2.0.0.yaml +``` + +The document is deliberately **not shipped in this chart**, for the same reason +the Dockerfile does not bake it into the image: a copy here is a second source of +truth that ages independently of the protocol. + +A volume is mounted at `becknSpec.cacheDir` either way — not only for the +air-gapped case. `/app` is not writable by uid 65532, and with +`readOnlyRootFilesystem` on, the fetch path would have nowhere to write its +cache. + +## Probes + +`/healthz` and `/readyz` answer different questions, and the split is +load-bearing: + +| Probe | Path | Behaviour | +|---|---|---| +| startup | `/healthz` | 60 × 5s. Covers the spec fetch, and migrations when they are on | +| liveness | `/healthz` | Depends on nothing. Stays 200 with the database down | +| readiness | `/readyz` | Pings PostgreSQL. 503 when it cannot reach it | + +Pointing liveness at `/readyz` would restart every pod during a database blip, +turning a recoverable outage into a crashloop across the whole Deployment. That +is why the two paths are different and why neither is configurable to the same +value by accident. + +There is no `exec` probe and no compose-style healthcheck: the runtime stage is +`gcr.io/distroless/static-debian12:nonroot`, which has no shell and no curl for +one to run. + +## Security context + +Both security contexts are **on by default**, unlike the other OAN service +charts. The image is distroless/static running as uid 65532 with a fully static +binary: there is no shell to escalate into, and the only path the process writes +is the spec cache, which is a mounted volume. `readOnlyRootFilesystem: true` +therefore costs nothing. + +If you add something that needs scratch space, add an `emptyDir` through +`extraVolumes` rather than turning this off. + +## Configuration + +`envConfig` is **empty by default**, and that is the deliberate part. + +The image already carries `config/common.yaml` — the project's reviewed defaults +for search sizing, validation, auth and the rest — and the environment layer sits +above it. Copying those values into this chart would create a second copy of +decisions the chart did not make, and it is the second copy that rots. + +Everything the chart genuinely owns is derived from structured values instead, so +`SERVER_PORT` cannot drift from the container port and `DATABASE_URL` cannot +drift from `database.*`: + +| Variable | Comes from | +|---|---| +| `DATABASE_URL` | `database.urlSecret`, or assembled from `database.host/port/name/user/sslMode` + `passwordSecret` | +| `DATABASE_AUTO_MIGRATE` | `database.autoMigrate` | +| `DATABASE_MAX_CONNS`, `DATABASE_MIN_CONNS` | `database.maxConns`, `database.minConns` | +| `APP_NETWORK_ID` | `app.networkId` — **required** | +| `APP_DEFAULT_TIMEZONE` | `app.defaultTimezone` | +| `SERVER_PORT` | `service.targetPort` | +| `LOG_LEVEL` | `logLevel` | +| `VALIDATION_SPEC_URL`, `VALIDATION_SPEC_CACHE_PATH` | `becknSpec.*` | +| `EMBEDDING_PROVIDER/MODEL/DIMENSIONS/ENDPOINT` | `embeddings.*` | +| `OTEL_EXPORTER`, `OTEL_EXPORTER_OTLP_ENDPOINT` | `otel.*` | +| `REPLICATION_TARGETS` | `replicationTargets`, joined with commas | + +Use `envConfig` to override a `common.yaml` default for one deployment. +Everything else the service reads: + +| Variable | Default | Notes | +|---|---|---| +| `SEARCH_DEFAULT_PAGE_SIZE` | 20 | What a request naming no limit gets | +| `SEARCH_MAX_PAGE_SIZE` | 100 | A larger limit is clamped to this | +| `SEARCH_MAX_CANDIDATES_PER_MODE` | 500 | Also the reachable pagination depth | +| `SEARCH_MAX_RADIUS_METERS` | 200000 | | +| `SEARCH_READ_DEADLINE` | 2s | | +| `SEARCH_FAIL_ON_UNAVAILABLE_MODE` | false | true turns a degraded mode into a 400 | +| `RATE_LIMIT_RPS` / `RATE_LIMIT_BURST` | 20 / 40 | `burst >= rps > 0`; a bucket smaller than one second's refill can never fill | +| `SERVER_SHUTDOWN_TIMEOUT` | 15s | | +| `SERVER_MAX_REQUEST_BODY_BYTES` | 10485760 | | +| `VALIDATION_ENABLE_L1_SCHEMA` | true | | +| `VALIDATION_ENABLE_L2_CONTEXT` | false | Nothing reads it; `true` refuses the boot | +| `AUTH_ENABLE_SIGNATURE_VERIFICATION` | false | Phase 2; `true` refuses the boot | +| `EXT_ALLOW_NETWORK_FETCH` | false | A configured URL is trusted, one from a request body is not | +| `GEO_RESOLUTION_CELLS` | 8 | H3 resolution | +| `ERROR_INCLUDE_LEGACY_TYPE` | false | | + +A key that matches no config field **fails the boot**, so a typo in `envConfig` +stops the service rather than being silently ignored. + +`envConfig` lands in a ConfigMap. Never put a secret there — use `secretEnv` or +`envFromSecrets`. + +## Differences from compose + +| | compose | chart | Why | +|---|---|---|---| +| `DATABASE_AUTO_MIGRATE` | `true` | `false` | Compose says it: "left false in deployment, where migrating is a step someone decides to take" | +| `RATE_LIMIT_RPS` / `BURST` | `100000` | unset (20 / 40) | Compose says it: "effectively off for the local stack… deployments leave both unset and get the real defaults back" | +| Beckn document | bind-mounted from the working tree | fetched, or from a ConfigMap | There is no working tree to mount from | +| `LOG_LEVEL` | `debug` | `info` | `debug` in the dev example | +| PostgreSQL | `pgvector/pgvector:0.8.0-pg16` | `dhi.io/pgvector:0.8-pg16` under CNPG | Same major and same pgvector minor line, so the planner behaviour the design doc measured still holds. CNPG additionally needs `vector` created at bootstrap — see [The database](#the-database) | +| healthcheck | none — distroless has no shell | HTTP probes | The compose file notes the probes are HTTP and "for you rather than for Compose" | +| Superuser | `POSTGRES_USER=discovery` **is** the superuser | the service owns its own database and is not superuser | This is why the `vector` extension problem appears only in the cluster: in compose the migration creates it as superuser without anyone noticing it needed to be one. The cluster creates it at bootstrap instead | + +## Validation + +```bash +../../scripts/lint-charts.sh +``` + +Two `ci/*-values.yaml` files render as separate cases, deliberately covering +opposite branches: `lint-values.yaml` is the assembled DSN with a fetched spec, +`url-secret-values.yaml` is the whole-DSN Secret with a ConfigMap spec, plus +autoscaling-adjacent extras, Ingress and a PDB. A chart that only ever renders +its defaults has untested branches that fail on the day someone uses them. + +Render-time guardrails, all of which name the value and the reason: + +- `image.repository` empty +- `app.networkId` empty +- neither `database.urlSecret.name` nor `database.passwordSecret.name` +- `database.host` empty while the DSN is assembled +- neither `becknSpec.url` nor `becknSpec.existingConfigMap` +- `embeddings.provider` other than `noop` with no `embeddings.endpoint` +- `otel.exporter` other than `none` with no `otel.endpoint` +- `resources` empty (`oan-common`) +- a probe with no handler or with two (`oan-common`) +- a PodDisruptionBudget alongside a single replica diff --git a/charts/discovery/ci/lint-values.yaml b/charts/discovery/ci/lint-values.yaml new file mode 100644 index 0000000..5491ee8 --- /dev/null +++ b/charts/discovery/ci/lint-values.yaml @@ -0,0 +1,20 @@ +# Minimum values that let `helm template` run in CI. Dummy references only - +# see examples/ for real per-environment configuration. +# +# Covers the assembled-DSN path (database.passwordSecret). The whole-DSN path +# is covered by ci/url-secret-values.yaml. +image: + registry: ghcr.io + repository: openagrinet/discovery-service + tag: "0.1.0" + +app: + networkId: local-network + +database: + host: discovery-db-rw + passwordSecret: + name: discovery-db-app + +becknSpec: + url: https://example.invalid/beckn.yaml diff --git a/charts/discovery/ci/url-secret-values.yaml b/charts/discovery/ci/url-secret-values.yaml new file mode 100644 index 0000000..01508db --- /dev/null +++ b/charts/discovery/ci/url-secret-values.yaml @@ -0,0 +1,53 @@ +# The other half of the matrix: the DSN comes whole from a Secret key, the spec +# from a ConfigMap rather than a URL, and the optional resources are all on. +# +# Rendering this is what keeps the alternative paths honest - a chart that only +# ever renders its defaults has untested branches that fail on the day someone +# uses them. +image: + registry: ghcr.io + repository: openagrinet/discovery-service + tag: "0.1.0" + +replicaCount: 2 + +app: + networkId: local-network + +database: + urlSecret: + name: discovery-db-app + key: uri + autoMigrate: true + +becknSpec: + existingConfigMap: discovery-beckn-spec + +embeddings: + provider: ollama + endpoint: http://ollama:11434 + +otel: + exporter: otlp + endpoint: http://otel-collector:4317 + +replicationTargets: + - https://peer-a.example.invalid/publish + - https://peer-b.example.invalid/publish + +ingress: + enabled: true + className: alb + hosts: + - host: discovery.example.invalid + paths: + - path: / + pathType: Prefix + +podDisruptionBudget: + enabled: true + minAvailable: 1 + +envConfig: + RATE_LIMIT_RPS: "50" + RATE_LIMIT_BURST: "100" diff --git a/charts/discovery/examples/discovery.dev.yaml b/charts/discovery/examples/discovery.dev.yaml new file mode 100644 index 0000000..48a0f8e --- /dev/null +++ b/charts/discovery/examples/discovery.dev.yaml @@ -0,0 +1,85 @@ +# Example: the OAN discover-and-publish service, dev environment. +# +# helm install discovery charts/discovery \ +# -n oan-discovery -f charts/discovery/examples/discovery.dev.yaml +# +# Install the database FIRST, and with an image that has pgvector - see +# README.md, "The database". Starting before the database is up is not fatal - +# the pod crashloops until it can connect, and Kubernetes' own backoff is the +# retry - but a database without pgvector never becomes reachable in the sense +# that matters, because the migration fails on the missing extension. + +# No fullnameOverride needed: the chart is named "discovery", so a release +# named "discovery" already produces Service/discovery. + +# TODO: no image is published for discovery-service yet - CI builds and scans +# one but pushes nothing. Fill this in with whatever the first published tag is; +# the render fails until then, on purpose. +image: + registry: ghcr.io + repository: openagrinet/discovery-service + tag: "0.1.0" + +app: + # The network this deployment serves. It fills an empty + # publishDirectives.visibleTo, so a catalog published without one is visible + # to this network and no other - which makes this the single most consequential + # value in the file. + networkId: mahavistar + +logLevel: debug + +database: + # CNPG generates Secret/-app with a ready-made `uri` key, so there is + # nothing to assemble and no password escaping to get wrong. The DSN carries + # host, port, database and user, so none of those are set here - a second copy + # would be one more thing to keep true. + urlSecret: + name: discovery-db-app + key: uri + + # Dev migrates on boot. Keep replicaCount at 1 while this is true: the + # migration runs in-process, so several pods starting at once race to apply + # the same one. + autoMigrate: true + + # Two retrieval modes today, one in-flight discover per mode. 32 is generous + # for dev and leaves room for the third mode when semantic search lands. + maxConns: 32 + minConns: 4 + +# Fetched at boot and cached in an emptyDir. For a cluster with no egress, or +# for a reproducible boot, create the ConfigMap instead: +# kubectl -n oan-discovery create configmap discovery-beckn-spec \ +# --from-file=beckn.yaml=/tests/testdata/beckn-v2.0.0.yaml +# and set existingConfigMap below. Note this URL tracks `main` and so pins +# nothing. +becknSpec: + url: https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/heads/main/api/v2.0.0/beckn.yaml + existingConfigMap: "" + +# Semantic search stays off until an Ollama deployment exists (A5). Discover +# still answers; it degrades the semantic mode and names it in X-Beckn-Degraded. +embeddings: + provider: noop + +otel: + exporter: none + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +ingress: + enabled: false + +# `helm test discovery -n oan-discovery` checks /readyz. +tests: + enabled: true + +commonLabels: + oan.in/environment: dev diff --git a/charts/discovery/examples/discovery.prod.yaml b/charts/discovery/examples/discovery.prod.yaml new file mode 100644 index 0000000..378d8fb --- /dev/null +++ b/charts/discovery/examples/discovery.prod.yaml @@ -0,0 +1,122 @@ +# Example: the OAN discover-and-publish service, production. +# +# helm install discovery charts/discovery \ +# -n oan-discovery -f charts/discovery/examples/discovery.prod.yaml +# +# NOT INSTALLABLE AS-IS. image.repository is empty and the render fails until +# it is filled in - see below. + +# --------------------------------------------------------------------------- +# TODO: the image OAN deploys in production is not decided. Prefer a DIGEST +# over a tag: a tag can be repointed at different bits, a digest cannot, which +# is what makes a rollback land on the same image it did before. +# +# image: +# registry: ghcr.io +# repository: openagrinet/discovery-service +# digest: sha256:... +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: "" + digest: "" + +replicaCount: 3 + +app: + networkId: mahavistar + +logLevel: info + +database: + urlSecret: + name: discovery-db-app + key: uri + + # OFF in production. Migrating is a step someone decides to take, on a + # release that was chosen for it - not something that happens because a pod + # was rescheduled at 3am. To migrate: + # + # helm upgrade discovery -n oan-discovery --reuse-values \ + # --set database.autoMigrate=true --set replicaCount=1 + # # verify, then upgrade back + autoMigrate: false + + # 3 replicas x 48 = 144 connections at full stretch. Check the cluster's + # max_connections before raising either number, and remember the autoscaler + # multiplies this by maxReplicas rather than by replicaCount. + maxConns: 48 + minConns: 8 + +# A ConfigMap rather than a fetch: production should not have its boot depend +# on GitHub being reachable, and the upstream URL tracks `main`, so a fetch +# pins nothing. Create it from a reviewed copy of the document: +# kubectl -n oan-discovery create configmap discovery-beckn-spec \ +# --from-file=beckn.yaml=beckn-v2.0.0.yaml +becknSpec: + url: "" + existingConfigMap: discovery-beckn-spec + key: beckn.yaml + +embeddings: + provider: noop + +# TODO: point at the cluster's collector once one is deployed. +otel: + exporter: none + endpoint: "" + +resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 1Gi + +ingress: + enabled: true + className: alb + annotations: + alb.ingress.kubernetes.io/scheme: internal + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/healthcheck-path: /readyz + hosts: + - host: discovery.oan.in + paths: + - path: / + pathType: Prefix + tls: [] + +# Safe with 3 replicas: at most one pod is disrupted at a time, so a node drain +# proceeds without taking the service below two. +podDisruptionBudget: + enabled: true + maxUnavailable: 1 + minAvailable: "" + +# Spread across nodes, so one node's loss is not the whole service. `preferred` +# rather than `required`: a hard rule leaves pods Pending when the cluster has +# fewer nodes than replicas, which turns a scale-up into an outage. +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: discovery + app.kubernetes.io/instance: discovery + +# Left OFF deliberately. Enabling it means capacity is decided by CPU, and the +# ceiling it puts on the database is maxReplicas x database.maxConns - raise +# them together, or not at all. +autoscaling: + enabled: false + +tests: + enabled: true + +commonLabels: + oan.in/environment: prod diff --git a/charts/discovery/templates/NOTES.txt b/charts/discovery/templates/NOTES.txt new file mode 100644 index 0000000..36ae0a4 --- /dev/null +++ b/charts/discovery/templates/NOTES.txt @@ -0,0 +1,59 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "discovery.fullname" . }} ({{ if .Values.autoscaling.enabled }}autoscaled {{ .Values.autoscaling.minReplicas }}-{{ .Values.autoscaling.maxReplicas }}{{ else }}{{ .Values.replicaCount }} replica(s){{ end }}) + - Service/{{ include "discovery.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "discovery.envConfigMapName" . }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "discovery.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "discovery.fullname" . }} +{{- end }} +{{- if .Values.podDisruptionBudget.enabled }} + - PodDisruptionBudget/{{ include "discovery.fullname" . }} +{{- end }} + +Wired to: + network: {{ .Values.app.networkId }} + database: {{ if .Values.database.urlSecret.name }}DSN from Secret/{{ .Values.database.urlSecret.name }} key {{ .Values.database.urlSecret.key }}{{ else }}{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.name }} (as {{ .Values.database.user }}){{ end }} + migrations: {{ if .Values.database.autoMigrate }}APPLIED ON BOOT{{ else }}not applied - DATABASE_AUTO_MIGRATE is false{{ end }} + beckn spec: {{ if .Values.becknSpec.url }}{{ .Values.becknSpec.url }}{{ else }}no URL - cache only{{ end }} + cache {{ if .Values.becknSpec.existingConfigMap }}from ConfigMap/{{ .Values.becknSpec.existingConfigMap }}{{ else }}in an emptyDir (refetched on every reschedule){{ end }} at {{ include "discovery.specCachePath" . }} + embeddings: {{ .Values.embeddings.provider }}{{ if eq .Values.embeddings.provider "noop" }} - semantic search is OFF, and discover says so in X-Beckn-Degraded{{ end }} + +In-cluster endpoint: + http://{{ include "discovery.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +Check it: + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "discovery.fullname" . }} + helm test {{ .Release.Name }} -n {{ .Release.Namespace }} + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "discovery.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl http://localhost:{{ .Values.service.port }}/healthz # liveness - depends on nothing + curl http://localhost:{{ .Values.service.port }}/readyz # readiness - 503 when PostgreSQL is unreachable + +{{ if not .Values.database.autoMigrate -}} +Migrations are OFF. On a database that has never been migrated, every request +fails against missing tables while /healthz still answers 200. To migrate: + + helm upgrade {{ .Release.Name }} -n {{ .Release.Namespace }} --reuse-values \ + --set database.autoMigrate=true --set replicaCount=1 + # then turn it back off once the rollout is complete + +Scale to one replica first: the migration runs in-process on boot, so several +pods starting at once race to apply it. +{{ end -}} + +If the pods crashloop before serving, check these in order - the first two look +nothing like their cause in the logs: + 1. "permission denied to create extension vector" means the database was not + bootstrapped with it. `vector` is not a trusted extension, so the owner + this service connects as cannot create it - the cluster must, at bootstrap, + as superuser. See charts/postgresql-cnpg/examples/discovery-db.dev.yaml. + "could not open extension control file" instead means the image has no + pgvector at all; no stock CNPG operand image does. + 2. "load the validation spec" means neither the fetch nor the cache produced + a Beckn document. With becknSpec.url set and no ConfigMap, the pod needs + egress to that URL. + 3. A key in `envConfig` matching no config field fails the boot by design, so + a typo there stops the service rather than being ignored. diff --git a/charts/discovery/templates/_helpers.tpl b/charts/discovery/templates/_helpers.tpl new file mode 100644 index 0000000..fc5fc4f --- /dev/null +++ b/charts/discovery/templates/_helpers.tpl @@ -0,0 +1,178 @@ +{{/* +# ============================================================================ +# DISCOVERY SERVICE CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the database, +# Beckn specification and environment wiring this service needs. +# ============================================================================ +*/}} + +{{- define "discovery.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "discovery.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "discovery.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "discovery.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "discovery.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "discovery.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "discovery.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +Absolute path of the Beckn spec cache, VALIDATION_SPEC_CACHE_PATH. + +Always absolute and always set explicitly, rather than left to the service's +own default of ".cache/beckn/beckn.yaml": that one is relative to the working +directory, which makes the path a fact about the image rather than about this +chart, and the volume mount below has to name a directory that agrees with it. +*/}} +{{- define "discovery.specCacheDir" -}} +{{- .Values.becknSpec.cacheDir | trimSuffix "/" -}} +{{- end }} + +{{- define "discovery.specCachePath" -}} +{{- printf "%s/%s" (include "discovery.specCacheDir" .) .Values.becknSpec.key -}} +{{- end }} + +{{/* +Emits "true" when the spec cache is backed by a ConfigMap rather than by an +emptyDir the fetched document is written into. +*/}} +{{- define "discovery.specFromConfigMap" -}} +{{- if .Values.becknSpec.existingConfigMap -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +The DSN, when this chart assembles one. Empty when database.urlSecret is set, +because then the whole DSN comes from that Secret instead. + +The password is NOT interpolated here - it is referenced as $(DATABASE_PASSWORD) +and expanded by the kubelet against the secretKeyRef env var defined just above +it in the container spec. That keeps the password out of the rendered manifest, +out of `helm get values`, and out of anything that logs a template. + +Because that expansion is textual, a password containing a URL delimiter +(@ : / ? # %) must already be percent-encoded in the Secret. urlSecret avoids +the question entirely, which is why it is the documented default. +*/}} +{{- define "discovery.databaseURL" -}} +{{- $db := .Values.database -}} +{{- printf "postgres://%s:$(DATABASE_PASSWORD)@%s:%v/%s?sslmode=%s" $db.user $db.host $db.port $db.name $db.sslMode -}} +{{- end }} + +{{/* +Everything the service needs that this chart derives from its structured +values, as container env entries. These take precedence over the envConfig +ConfigMap injected with envFrom, which is what makes "derived here, overridable +there" safe rather than ambiguous. + +Every guardrail below refuses to render something the service would refuse to +boot on. The failure is the same either way; the difference is that this one +names the value and happens before anything is applied to the cluster. +*/}} +{{- define "discovery.env" -}} +{{- $db := .Values.database -}} +{{- $spec := .Values.becknSpec -}} +{{- if not .Values.app.networkId }} +{{- fail (printf "%s: app.networkId is required - it is APP_NETWORK_ID, which the boot refuses without, and it decides who a catalog published with no explicit visibleTo is visible to. Set the network this deployment serves, e.g. mahavistar." .Chart.Name) }} +{{- end }} +{{- if and (not $db.urlSecret.name) (not $db.passwordSecret.name) }} +{{- fail (printf "%s: the database DSN is unset. Set database.urlSecret.name (preferred - CNPG writes a ready-made `uri` key into Secret/-app), or database.passwordSecret.name together with database.host/name/user to have the chart assemble one. This chart renders no Secrets." .Chart.Name) }} +{{- end }} +{{- if and (not $db.urlSecret.name) (not $db.host) }} +{{- fail (printf "%s: database.host is required when the DSN is assembled - point it at the PostgreSQL primary service, e.g. discovery-db-rw" .Chart.Name) }} +{{- end }} +{{- if and (not $spec.url) (not $spec.existingConfigMap) }} +{{- fail (printf "%s: set becknSpec.url or becknSpec.existingConfigMap. The service loads the Beckn document before it serves and refuses to start without it, and with neither set there is nothing to fetch and nothing cached to fall back to." .Chart.Name) }} +{{- end }} +{{- if not $spec.key }} +{{- fail (printf "%s: becknSpec.key is required - it is both the key read from the ConfigMap and the filename the cache is written under" .Chart.Name) }} +{{- end }} +{{- if $db.urlSecret.name -}} +- name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ $db.urlSecret.name }} + key: {{ $db.urlSecret.key }} +{{- else }} +{{- /* Defined FIRST: the kubelet expands $(VAR) only against variables that + precede it in this list. */ -}} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $db.passwordSecret.name }} + key: {{ $db.passwordSecret.key }} +- name: DATABASE_URL + value: {{ include "discovery.databaseURL" . | quote }} +{{- end }} +- name: DATABASE_AUTO_MIGRATE + value: {{ $db.autoMigrate | quote }} +- name: DATABASE_MAX_CONNS + value: {{ $db.maxConns | quote }} +- name: DATABASE_MIN_CONNS + value: {{ $db.minConns | quote }} +- name: APP_NETWORK_ID + value: {{ .Values.app.networkId | quote }} +- name: APP_DEFAULT_TIMEZONE + value: {{ .Values.app.defaultTimezone | quote }} +{{- /* Derived from the port the container actually publishes, so the two + cannot drift into a Service that routes to a port nothing listens on. */}} +- name: SERVER_PORT + value: {{ .Values.service.targetPort | quote }} +- name: LOG_LEVEL + value: {{ .Values.logLevel | quote }} +- name: VALIDATION_SPEC_CACHE_PATH + value: {{ include "discovery.specCachePath" . | quote }} +{{- with $spec.url }} +- name: VALIDATION_SPEC_URL + value: {{ . | quote }} +{{- end }} +- name: EMBEDDING_PROVIDER + value: {{ .Values.embeddings.provider | quote }} +{{- if ne .Values.embeddings.provider "noop" }} +- name: EMBEDDING_MODEL + value: {{ .Values.embeddings.model | quote }} +- name: EMBEDDING_DIMENSIONS + value: {{ .Values.embeddings.dimensions | quote }} +{{- if not .Values.embeddings.endpoint }} +{{- fail (printf "%s: embeddings.provider is %q, so embeddings.endpoint is required - a provider with nowhere to call fails at the first publish, not at boot" .Chart.Name .Values.embeddings.provider) }} +{{- end }} +- name: EMBEDDING_ENDPOINT + value: {{ .Values.embeddings.endpoint | quote }} +{{- end }} +- name: OTEL_EXPORTER + value: {{ .Values.otel.exporter | quote }} +{{- if ne .Values.otel.exporter "none" }} +{{- if not .Values.otel.endpoint }} +{{- fail (printf "%s: otel.exporter is %q, so otel.endpoint is required - set it to the collector, e.g. http://otel-collector:4317" .Chart.Name .Values.otel.exporter) }} +{{- end }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} +{{- with .Values.replicationTargets }} +- name: REPLICATION_TARGETS + value: {{ join "," . | quote }} +{{- end }} +{{- with (include "oan-common.env" . | trim) }} +{{ . }} +{{- end }} +{{- end }} diff --git a/charts/discovery/templates/configmap.yaml b/charts/discovery/templates/configmap.yaml new file mode 100644 index 0000000..eea3b40 --- /dev/null +++ b/charts/discovery/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "discovery.envConfigMapName" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with (include "oan-common.envConfigMapData" . | trim) }} +data: + {{- . | nindent 2 }} +{{- else }} +data: {} +{{- end }} diff --git a/charts/discovery/templates/deployment.yaml b/charts/discovery/templates/deployment.yaml new file mode 100644 index 0000000..82b37b3 --- /dev/null +++ b/charts/discovery/templates/deployment.yaml @@ -0,0 +1,111 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "discovery.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "discovery.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "discovery.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "discovery.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- include "discovery.env" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "discovery.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + {{- /* + The Beckn spec cache. Mounted either way, and not only for the + air-gapped case: with readOnlyRootFilesystem the fetch path has + nowhere to write its cache without a volume here, and /app is not + writable by uid 65532 in any case. + */}} + - name: beckn-spec + mountPath: {{ include "discovery.specCacheDir" . }} + readOnly: {{ if include "discovery.specFromConfigMap" . }}true{{ else }}false{{ end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: beckn-spec + {{- if include "discovery.specFromConfigMap" . }} + configMap: + name: {{ .Values.becknSpec.existingConfigMap }} + items: + - key: {{ .Values.becknSpec.key }} + path: {{ .Values.becknSpec.key }} + {{- else }} + emptyDir: + sizeLimit: 16Mi + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/discovery/templates/hpa.yaml b/charts/discovery/templates/hpa.yaml new file mode 100644 index 0000000..9687662 --- /dev/null +++ b/charts/discovery/templates/hpa.yaml @@ -0,0 +1,54 @@ +{{- if .Values.autoscaling.enabled }} +{{/* +The service is stateless - it holds no session state and writes everything to +PostgreSQL - so it scales horizontally. Two things to keep in mind: + + - Every replica opens its own pool of up to database.maxConns connections, so + the ceiling this chart puts on the server is maxReplicas x maxConns. Raise + the two together, against the server's `max_connections`, not separately. + - Autoscaling takes over `replicas`, so replicaCount is ignored once this is + enabled. The Deployment deliberately omits `replicas` in that case, or the + two would fight on every reconcile. + +Do NOT enable this for the install that migrates: DATABASE_AUTO_MIGRATE with +several pods starting at once has them race to apply the same migration. +*/}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "discovery.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- with .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/discovery/templates/ingress.yaml b/charts/discovery/templates/ingress.yaml new file mode 100644 index 0000000..c6c9dfd --- /dev/null +++ b/charts/discovery/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "discovery.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/discovery/templates/poddisruptionbudget.yaml b/charts/discovery/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..1fa8a56 --- /dev/null +++ b/charts/discovery/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Caps how many pods can be voluntarily disrupted at once - node drains, cluster +upgrades, autoscaler scale-down. + +With replicaCount: 1 a PDB of minAvailable: 1 blocks drains entirely, because +evicting the only pod would breach it. That is why this is disabled by default +and should be enabled together with replicaCount >= 2. +*/}} +{{- if and (eq (int .Values.replicaCount) 1) (not .Values.podDisruptionBudget.allowSingleReplica) }} +{{- fail (printf "%s: podDisruptionBudget is enabled with replicaCount 1, which blocks node drains entirely. Raise replicaCount to 2+, or set podDisruptionBudget.allowSingleReplica=true if you accept that." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "discovery.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/discovery/templates/service.yaml b/charts/discovery/templates/service.yaml new file mode 100644 index 0000000..37a301b --- /dev/null +++ b/charts/discovery/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "discovery.fullname" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "discovery.selectorLabels" . | nindent 4 }} diff --git a/charts/discovery/templates/serviceaccount.yaml b/charts/discovery/templates/serviceaccount.yaml new file mode 100644 index 0000000..90c4e8c --- /dev/null +++ b/charts/discovery/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "discovery.serviceAccountName" . }} + labels: + {{- include "discovery.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/discovery/templates/tests/test-connection.yaml b/charts/discovery/templates/tests/test-connection.yaml new file mode 100644 index 0000000..7492d2f --- /dev/null +++ b/charts/discovery/templates/tests/test-connection.yaml @@ -0,0 +1,40 @@ +{{- if .Values.tests.enabled }} +{{/* +`helm test` check: the service answers on /readyz. + +/readyz rather than /healthz on purpose. /healthz depends on nothing and stays +200 with the database down, so a test against it would pass on a deployment +that cannot serve a single discover. /readyz pings the datastore, which is the +question worth asking after an install. + +Deliberately does not POST a discover: that needs a published catalog to find, +and a failure would be ambiguous between "the service is broken" and "nothing +has been published yet". +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "discovery.fullname" . }}-test-connection + labels: + {{- include "discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: test + image: {{ printf "%s/%s:%s" .Values.tests.image.registry .Values.tests.image.repository .Values.tests.image.tag | quote }} + command: + - /bin/sh + - -c + - | + set -e + url="http://{{ include "discovery.fullname" . }}:{{ .Values.service.port }}/readyz" + echo "GET $url" + wget -q -O- --timeout=10 "$url" + echo "" + echo "OK: discovery is ready - it is serving and it can reach PostgreSQL" + resources: + {{- toYaml .Values.tests.resources | nindent 8 }} +{{- end }} diff --git a/charts/discovery/values.yaml b/charts/discovery/values.yaml new file mode 100644 index 0000000..90c698f --- /dev/null +++ b/charts/discovery/values.yaml @@ -0,0 +1,416 @@ +# ============================================================================ +# discovery - default values +# +# The OAN Beckn v2.0.0 discover-and-publish service. Defaults mirror the +# verified compose stack (discovery-service/docker-compose.yml), with the two +# settings that file marks as local-only put back to their deployment values: +# DATABASE_AUTO_MIGRATE and the rate limiter. +# +# Depends on one thing already running: +# - PostgreSQL, WITH the `vector` and `pg_trgm` extensions -> database.* +# +# Minimum you must set: +# - image.repository +# - app.networkId +# - database.host, and either database.urlSecret.name or +# database.passwordSecret.name +# - becknSpec.url or becknSpec.existingConfigMap +# +# The render fails, with the reason, if any of those is missing. That is +# deliberate: every one of them is something the service refuses to boot +# without, and a pod that CrashLoopBackOffs reports the problem far later and +# far less clearly than `helm template` does. +# ============================================================================ + +replicaCount: 1 + +# --------------------------------------------------------------------------- +# Image +# +# repository is EMPTY on purpose. discovery-service's CI builds an image and +# scans it but pushes it nowhere, so there is no tag to default to; the compose +# stack builds from the working tree. The render fails while this is empty +# rather than producing "ghcr.io/:0.1.0", which Helm and the API server both +# accept and which only surfaces later as an ImagePullBackOff. +# +# The runtime is gcr.io/distroless/static-debian12:nonroot - no shell, no +# package manager, uid 65532. That is what makes the security context defaults +# below safe to leave on. +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: "" + tag: "" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: false + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + annotations: {} + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: discovery.local + paths: + - path: / + pathType: Prefix + tls: [] + +# --------------------------------------------------------------------------- +# Application +# --------------------------------------------------------------------------- +app: + # APP_NETWORK_ID. REQUIRED - the boot refuses without it, so the render does + # too. It fills an empty publishDirectives.visibleTo (C8), which means every + # catalog published without one is visible to this network and no other. + # There is no sensible default: "mahavistar", "bharatvistar" and the rest are + # different networks, and guessing wrong publishes to the wrong audience. + networkId: "" + # APP_DEFAULT_TIMEZONE. Validated with time.LoadLocation at startup - a typo + # fails the boot rather than silently shifting every daily validity window. + defaultTimezone: Asia/Kolkata + +# LOG_LEVEL +logLevel: info + +# --------------------------------------------------------------------------- +# Database +# +# The service needs PostgreSQL with two extensions its first migration creates: +# +# CREATE EXTENSION IF NOT EXISTS vector; -- pgvector, for the embedding column and its HNSW index +# CREATE EXTENSION IF NOT EXISTS pg_trgm; -- contrib, ships with every ordinary PostgreSQL build +# +# pg_trgm is contrib and is present in the CloudNativePG operand images. +# pgvector is NOT: the stock CNPG image cannot create the `vector` extension, +# and the migration - and therefore the boot - fails on it. Point +# postgresql-cnpg's `image.*` at an operand image that bundles pgvector before +# installing this chart. See README.md, "The database". +# +# Two ways to supply the DSN, and exactly one of them must be set: +# +# urlSecret one Secret key holding the whole DATABASE_URL. PREFERRED: +# CNPG already writes a ready-made `uri` key into the app +# Secret it generates, so there is nothing to assemble and no +# escaping to get wrong. +# +# host/port/name/user + passwordSecret +# the DSN is assembled from these, with the password injected +# through Kubernetes' own $(VAR) expansion so it stays in the +# Secret and never lands in a ConfigMap. Note the caveat: the +# password is substituted verbatim into a URL, so a password +# containing @ : / ? # or % must already be percent-encoded +# in the Secret. +# +# urlSecret makes host/port/name/user/sslMode unread - the DSN carries all of +# it - so leave them empty rather than setting a second copy that nothing +# consumes and nothing keeps true. +# --------------------------------------------------------------------------- +database: + # PostgreSQL primary service, e.g. "discovery-db-rw" + host: "" + port: 5432 + name: discovery + user: discovery + # disable | allow | prefer | require | verify-ca | verify-full. Only used + # when the DSN is assembled. + sslMode: prefer + # Whole-DSN Secret. Wins over the assembled form when set. + urlSecret: + name: "" + # CNPG writes `uri` into Secret/-app + key: uri + # Password Secret, for the assembled form. + passwordSecret: + name: "" + key: password + + # DATABASE_AUTO_MIGRATE. The migrations are compiled into the binary, so this + # needs no sidecar and no mounted directory - the service applies them on + # boot and treats "already at the latest version" as success. + # + # False here, and true in compose, is not a discrepancy: migrating is a step + # someone decides to take. Turn it on for the install that is meant to + # migrate, and note that with replicaCount > 1 several pods race to do it - + # scale to 1, migrate, scale back. + autoMigrate: false + + # Sized by the concurrency model, not guessed: discover runs its retrieval + # modes concurrently (A2), so one in-flight discover holds as many + # connections as it has enabled modes. + # + # maxConns >= (enabled modes) x (expected in-flight discovers) + # + # Two modes today, three once semantic lands. Multiply by replicaCount and + # keep the total under the server's max_connections. minConns is a warm-start + # knob only - idle backends cost the server memory to save a handshake. + maxConns: 32 + minConns: 4 + +# --------------------------------------------------------------------------- +# Beckn specification +# +# The service loads the Beckn v2.0.0 document BEFORE it serves and refuses to +# start without it. The check is unconditional: turning L1 validation off does +# not remove the requirement. +# +# It tries `url` first and falls back to the on-disk cache, so at least one of +# `url` and `existingConfigMap` must be set or the boot cannot succeed - the +# render fails when both are empty. +# +# The document is deliberately NOT shipped in this chart, for the same reason +# the Dockerfile does not bake it into the image: a copy here is a second +# source of truth that ages independently of the protocol. +# --------------------------------------------------------------------------- +becknSpec: + # VALIDATION_SPEC_URL. Upstream is + # https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/heads/main/api/v2.0.0/beckn.yaml + # which tracks `main` and therefore pins nothing. Prefer a pinned URL, or the + # ConfigMap below, for a reproducible boot. + url: "" + # A ConfigMap holding the document, mounted as the cache. Set this for an + # air-gapped deploy, or alongside `url` as the fallback when the fetch fails. + # kubectl -n create configmap discovery-beckn-spec \ + # --from-file=beckn.yaml=tests/testdata/beckn-v2.0.0.yaml + existingConfigMap: "" + # Key inside that ConfigMap, and the filename it is mounted under. + key: beckn.yaml + # Directory the cache lives in. VALIDATION_SPEC_CACHE_PATH is this plus key. + # An emptyDir is mounted here when no ConfigMap is supplied, so the fetched + # document can be cached - /app is not writable by uid 65532. + cacheDir: /app/.cache/beckn + +# --------------------------------------------------------------------------- +# Embeddings +# +# Semantic search is deferred (A5): the column, the HNSW index and the Embedder +# seam all ship, the provider does not. `noop` is the reviewed default and the +# one Phase 1 runs on - a discover request simply degrades and says so in +# X-Beckn-Degraded (C11). +# +# Set provider: ollama and an endpoint once an Ollama deployment exists. +# dimensions must match the column the migration created; changing it on a +# populated database is a reindex, not a config change. +# --------------------------------------------------------------------------- +embeddings: + provider: noop + model: nomic-embed-text + endpoint: "" + dimensions: 768 + +# --------------------------------------------------------------------------- +# OpenTelemetry +# +# `none` boots without a collector (T2). Set exporter: otlp and an endpoint to +# export. +# --------------------------------------------------------------------------- +otel: + exporter: none + endpoint: "" + +# REPLICATION_TARGETS - peer discovery endpoints a publish is fanned out to. +# Rendered as the comma-separated list the service parses. +replicationTargets: [] + +resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + +# --------------------------------------------------------------------------- +# Probes +# +# /healthz and /readyz answer different questions, and the split is load-bearing +# (src/app/router.go): /healthz depends on nothing and stays 200 with the +# database down, /readyz pings the database and returns 503 when it cannot. +# Pointing liveness at /readyz would restart every pod during a database blip - +# turning a recoverable outage into a crashloop across the whole Deployment. +# +# There is no `exec` option and no compose-style healthcheck: the runtime stage +# is distroless/static, which has no shell for one to run. +# --------------------------------------------------------------------------- +startupProbe: + enabled: true + httpGet: + path: /healthz + port: http + # Generous: the boot fetches and compiles the Beckn document, and may apply + # migrations, before it listens. + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + +livenessProbe: + enabled: true + httpGet: + path: /healthz + port: http + initialDelaySeconds: 0 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + +readinessProbe: + enabled: true + httpGet: + path: /readyz + port: http + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# --------------------------------------------------------------------------- +# Security contexts +# +# On by default here, unlike the other OAN service charts. The image is +# distroless/static running as uid 65532 with a fully static binary: there is +# no shell to escalate into, and the only path the process writes is the spec +# cache, which is a mounted volume. readOnlyRootFilesystem therefore costs +# nothing - and if you add anything that needs scratch space, add an emptyDir +# through extraVolumes rather than turning this off. +# --------------------------------------------------------------------------- +podSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + +securityContext: + enabled: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# Extra init containers. Rendered verbatim. +extraInitContainers: [] + +# Extra volumes and mounts, rendered verbatim. +extraVolumes: [] +extraVolumeMounts: [] + +# --------------------------------------------------------------------------- +# PodDisruptionBudget +# +# Caps voluntary disruptions - node drains, cluster upgrades, autoscaler +# scale-down. Disabled by default because with a single replica a PDB of +# minAvailable: 1 blocks drains entirely; the render fails if you enable it +# anyway without acknowledging that. +# --------------------------------------------------------------------------- +podDisruptionBudget: + enabled: false + minAvailable: 1 + # Set one or the other, not both. + maxUnavailable: "" + # Enable a PDB with a single replica anyway, accepting that node drains block. + allowSingleReplica: false + +# --------------------------------------------------------------------------- +# Autoscaling +# +# The service is stateless - everything is in PostgreSQL - so it scales +# horizontally. Every replica opens its own pool of up to database.maxConns +# connections, so raise maxReplicas and maxConns together against the server's +# max_connections, not separately. When enabled, replicaCount is ignored. +# --------------------------------------------------------------------------- +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + # Set to a number to also scale on memory. + targetMemoryUtilizationPercentage: "" + # Scale-up/scale-down tuning, rendered verbatim into spec.behavior. + behavior: {} + +# --------------------------------------------------------------------------- +# `helm test` checks +# +# helm test -n +# +# Runs after install to confirm the service actually answers - which is the +# difference between "the pod is running" and "the deployment works". It checks +# /readyz rather than /healthz, because /healthz is 200 with the database down +# and would pass on a deployment that cannot serve a single request. +# --------------------------------------------------------------------------- +tests: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + +# --------------------------------------------------------------------------- +# ENVIRONMENT CONFIGURATION +# +# Rendered into a ConfigMap and injected with envFrom. EMPTY on purpose. +# +# The image already carries config/common.yaml - the project's reviewed +# defaults for search sizing, validation, auth and the rest - and the +# environment layer sits above it. Copying those values here would create a +# second copy of decisions this chart did not make, and it is the second copy +# that rots. Everything the chart genuinely owns is derived from the structured +# values above instead, so nothing here can drift from them. +# +# Use this to override a common.yaml default for one deployment. The full list +# is in README.md; the ones most likely to be wanted: +# +# SEARCH_DEFAULT_PAGE_SIZE, SEARCH_MAX_PAGE_SIZE, SEARCH_MAX_RADIUS_METERS +# SEARCH_MAX_CANDIDATES_PER_MODE, SEARCH_READ_DEADLINE +# SEARCH_FAIL_ON_UNAVAILABLE_MODE +# RATE_LIMIT_RPS, RATE_LIMIT_BURST (defaults 20 / 40; burst >= rps > 0) +# SERVER_SHUTDOWN_TIMEOUT, SERVER_MAX_REQUEST_BODY_BYTES +# VALIDATION_ENABLE_L1_SCHEMA, GEO_RESOLUTION_CELLS +# +# A key that matches no Config field fails the boot, so a typo here cannot +# silently do nothing. +# +# NEVER put secrets here. This is a ConfigMap. +# --------------------------------------------------------------------------- +envConfig: {} + +# Extra env vars from specific Secret keys, beyond DATABASE_URL. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} diff --git a/charts/registry/.helmignore b/charts/registry/.helmignore new file mode 100644 index 0000000..3027bb2 --- /dev/null +++ b/charts/registry/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.tmproj +.vscode/ +.idea/ +*.swp +*.bak +*.orig + +# Example values are documentation, not part of the package +examples/ diff --git a/charts/registry/CHANGELOG.md b/charts/registry/CHANGELOG.md new file mode 100644 index 0000000..3aaa256 --- /dev/null +++ b/charts/registry/CHANGELOG.md @@ -0,0 +1,74 @@ +# Changelog + +All notable changes to the `registry` chart are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] - 2026-08-31 + +### Changed +- `image.repository` now defaults to empty rather than + `sunbird-rc/sunbird-rc-core`. The image OAN will deploy is not settled, and a + default was reading as a decision that had not been made. The render fails + while it is empty, so this cannot be missed. The dev example sets the + compose-verified image explicitly; the production example is a documented TODO. + +### Removed +- External Secrets Operator integration: the `ExternalSecret` template and the + `externalSecrets` value block. ESO is not installed in any OAN cluster, so this + was configuration that could not be exercised, and a chart that renders a + Secret-producing resource invites the question of where secrets come from to be + answered differently per chart. + + Charts still reference Secrets by name - `envFromSecrets`, `secretEnv`, and the + per-chart `*Secret.name` settings are unchanged. Creating those Secrets is now + unambiguously outside the charts. + +## [0.1.0] - 2026-08-31 + +Initial release. Configuration ported from the verified +`registry/docker-compose.yml` stack, with the deployment shape taken from +Sunbird's own Helm charts (`deploy-as-code/helm/v2`). + +### Added +- Deployment, Service, env ConfigMap, schemas ConfigMap, ServiceAccount, and + optional Ingress and ESO ExternalSecret, built on the `oan-common` library. +- All 32 registry environment variables from the compose stack, verified at + parity. Database and Keycloak variables are derived from structured values + rather than restated, so `OAUTH2_RESOURCES_0_URI` cannot drift from + `keycloak.url` and `connectionInfo_uri` cannot drift from `database.*`. +- Entity schemas shipped in `files/schemas/` and rendered into a ConfigMap + mounted read-only, with `existingConfigMap` and `inline` overrides. A + `checksum/schemas` annotation rolls the pod when a schema changes. +- Render-time validation of `database.host`, `database.passwordSecret.name`, + `keycloak.url`, `keycloak.adminClientSecret.name`, and + `defaultUserPasswordSecret.name` when `keycloakUserSetPassword` is true. + Also fails when no schema file matches, since a registry with no entity + definitions serves nothing. +- Every optional Sunbird RC subsystem off by default, matching compose: + encryption, events, idgen, claims, DIDs, signatures, certificates, file + storage, notifications, async and webhooks. + +- Init containers that wait for the database and for Keycloak's realm endpoint. + The Keycloak check is HTTP against the realm rather than TCP against the port, + because the realm is imported during Keycloak's first start and the port opens + well before the realm exists. +- Optional HorizontalPodAutoscaler. `replicas` is omitted from the Deployment + when it is enabled, so the two do not fight on every reconcile. +- Optional PodDisruptionBudget, which fails the render if enabled alongside a + single replica. +- `helm test` check that `/health` responds. +- `extraVolumes` / `extraVolumeMounts` / `extraInitContainers` escape hatches. +- Per-environment example values for dev and production. The production example + connects as the database owner rather than the superuser, pulls secrets from + ESO, sets a PodDisruptionBudget and pod anti-affinity, and enables the pod and + container security contexts. + +### Changed from the compose stack +- Probes use `/health` (as compose does) rather than Sunbird's + `/api/docs/swagger.json`, which would also require swagger to be enabled. + A `startupProbe` covers first-start schema migration. +- Resource limits added; Sunbird's chart sets requests only. +- The three secret values are read from named Secret keys instead of `.env` + variables, and the chart renders none of them. diff --git a/charts/registry/Chart.yaml b/charts/registry/Chart.yaml new file mode 100644 index 0000000..1b9ab4c --- /dev/null +++ b/charts/registry/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: registry +description: >- + The OAN participant registry, on Sunbird RC core. Holds participant records + and the key material inbound signed requests are verified against. Requires + PostgreSQL and Keycloak. +type: application +version: 0.2.0 +appVersion: "v2.0.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - registry + - sunbird-rc + - participant +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/Sunbird-RC/sunbird-rc-core +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/registry/README.md b/charts/registry/README.md new file mode 100644 index 0000000..9c38797 --- /dev/null +++ b/charts/registry/README.md @@ -0,0 +1,244 @@ +# registry + +The OAN participant registry, on [Sunbird RC +core](https://github.com/Sunbird-RC/sunbird-rc-core). It holds participant +records and the key material that inbound signed requests are verified against. + +Configuration is ported from the verified `registry/docker-compose.yml` stack — +all 32 environment variables, at parity. This deployment is **participant records +and lookup only**: every optional Sunbird RC subsystem (credentials, DIDs, +certificates, encryption, file storage, notifications, webhooks, async) is off, +exactly as in compose. + +## What it renders + +| Resource | Notes | +|---|---| +| Deployment | Probes on `/health`, resources mandatory, init containers wait for dependencies | +| Service | `ClusterIP` on 8081 | +| ConfigMap (env) | The non-secret flags from `envConfig` | +| ConfigMap (schemas) | Entity definitions, mounted read-only | +| ServiceAccount | | +| HorizontalPodAutoscaler | Optional, off by default | +| PodDisruptionBudget | Optional, off by default | +| Ingress | Optional, off by default | +| Test Pod | `helm test` check against `/health` | + +## Dependencies + +Two things must already be running: + +1. **PostgreSQL** → `database.host`. See [`postgresql-cnpg`](../postgresql-cnpg), + which creates the database and the owner role this chart connects as. +2. **Keycloak** → `keycloak.url`. See + [`keycloak`](../keycloak). + +## Image + +`image.repository` is **empty by default, on purpose.** The image OAN will +actually deploy is not decided yet — upstream `ghcr.io`, a mirror in our own +registry, or an OAN-built image — and a default here would read as a decision +that has not been made. + +The render **fails** while it is empty: + +``` +registry: image.repository is required - set image.registry/repository/tag for this environment +``` + +That is deliberate. An empty repository would otherwise render +`image: ghcr.io/:v2.0.0`, which Helm and the API server both accept, and which +only fails later as an `ImagePullBackOff`. + +| Values file | Image | +|---|---| +| `examples/registry.dev.yaml` | `ghcr.io/sunbird-rc/sunbird-rc-core:v2.0.0` — what the compose stack runs, so dev deploys what was verified locally | +| `examples/registry.prod.yaml` | Empty, with a TODO. Not installable until filled in | + +When you do fill in production, prefer `image.digest` over `image.tag`: a tag can +be repointed at different bits, a digest cannot, which is what makes a rollback +land on the same image it did before. + +## Waiting for dependencies + +Kubernetes has no equivalent of compose's `depends_on: condition: +service_healthy`, so the chart adds init containers: + +```yaml +waitFor: + enabled: true + database: true # TCP check, host/port from `database` + keycloak: true # HTTP check on /realms/ +``` + +Both are derived from settings you already set, so there is nothing to keep in +sync. + +The Keycloak check is deliberately **HTTP against the realm endpoint**, not TCP +against the port. Keycloak imports the realm during its first start, so the port +opens well before the realm exists — a TCP check would pass too early and the +registry would come up against a Keycloak that cannot yet issue it a usable +token. + +They give up after `waitFor.timeoutSeconds` (default 300) so a genuinely missing +dependency shows as a failed init container in `kubectl describe`, rather than an +endless wait. + +## Verifying an install + +```bash +helm test registry -n oan-registry +``` + +Runs a Pod that checks `/health`. It deliberately does not test an authenticated +endpoint: that needs a real token, and a failure there would be ambiguous between +"the registry is broken" and "the Keycloak wiring is wrong". + +## Scaling + +The registry is stateless, so `autoscaling.enabled` gives it an HPA. Two things +to know: + +- Every replica opens its own connection pool, so scaling out multiplies + connections against PostgreSQL. Check the cluster's `max_connections` before + raising `maxReplicas` much. +- When autoscaling is on, the Deployment omits `replicas` entirely, so + `replicaCount` is ignored. Otherwise Helm and the HPA fight on every reconcile. + +`podDisruptionBudget.enabled` keeps a replica serving through node drains. It +**fails the render** if enabled with `replicaCount: 1`, because a PDB of +`minAvailable: 1` over a single pod blocks drains completely — set +`allowSingleReplica: true` if you want that anyway. + +## Per-environment values + +| File | For | +|---|---| +| [`examples/registry.dev.yaml`](./examples/registry.dev.yaml) | Dev: superuser connection, hand-made Secrets, no ingress | +| [`examples/registry.prod.yaml`](./examples/registry.prod.yaml) | Production: owner connection, 2 replicas, PDB, anti-affinity, security contexts on | + +Everything that differs between environments lives in these files, not in +templates. + +## Install order + +The full stack, in the only order that works: + +```bash +# 1. Database cluster. Creates BOTH databases - `registry` via bootstrap.initdb, +# `keycloak` via a CNPG Database object - each owned by its own role. +helm install registry-db charts/postgresql-cnpg \ + -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml + +# 2. Keycloak — imports the sunbird-rc realm on first start +helm install keycloak charts/keycloak \ + -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml + +# 3. MANUAL: regenerate the admin-api client secret. +# The realm export ships it masked ("**********"), so it does not work as-is. +# kubectl -n oan-registry port-forward svc/keycloak 8080:8080 +# http://localhost:8080/auth/admin -> realm sunbird-rc -> Clients +# -> admin-api -> Credentials -> Regenerate Secret +# kubectl -n oan-registry create secret generic registry-keycloak \ +# --from-literal=keycloakAdminClientSecret='' \ +# --from-literal=registryDefaultUserPassword='' + +# 4. Registry +helm install registry charts/registry \ + -n oan-registry -f charts/registry/examples/registry.dev.yaml +``` + +Step 3 is unavoidable while the realm is imported from a masked export — see +[`keycloak`](../keycloak#the-two-phase-first-install). + +## Entity schemas + +The registry treats every `.json` under its schema directory as an entity it +serves APIs for. The chart ships `files/schemas/Participant.json` and mounts the +rendered ConfigMap read-only at +`/home/sunbirdrc/config/public/_schemas`. A `checksum/schemas` annotation rolls +the pod when a schema changes. + +| You want | Set | +|---|---| +| The shipped schemas | nothing — this is the default | +| Schemas managed outside the chart | `schemas.existingConfigMap: ` | +| Extra or overriding schemas | `schemas.inline: {Name.json: ''}` | + +The render fails if no schema matches, since a registry with no entity +definitions serves nothing. + +> **The participant schema is not settled.** Per engineering-tracker #33, it was +> built and verified against the local stack but has no design issue of its own +> (#43 was closed as a duplicate of #69, which covers a different schema). +> Expect it to change, and a schema change is a chart release. + +## Authentication + +`authenticationEnabled: true` — role checks on `Participant` only apply while +this is true. Turning it off makes every endpoint unauthenticated; it is a +local-debugging switch, not a deployment option. + +Three settings must agree with Keycloak, and getting any of them wrong produces +401/403 responses that read like a permissions bug rather than a configuration +one: + +| Setting | Must be | +|---|---| +| `keycloak.url` | Exactly the issuer Keycloak puts in the `iss` claim, including `/auth` | +| `keycloak.realm` | A realm that is actually imported (`sunbird-rc`) | +| `keycloak.adminClientSecret` | The **regenerated** `admin-api` secret, never the masked one from the export | + +`OAUTH2_RESOURCES_0_URI` is derived as `/realms/`, so it +cannot drift from `keycloak.url`. + +## Database + +Defaults match compose: the registry connects to database `registry` as the +`postgres` superuser, sharing that database with Keycloak. + +**CNPG disables superuser access by default**, so reproducing the compose +arrangement requires `enableSuperuserAccess: true` on the database chart — which +the example values set, and which makes CNPG generate +`Secret/-superuser` for the password. + +Once you are past reproducing compose, the better arrangement is: leave +superuser access off, connect the registry as the cluster's bootstrap owner +(`database.user: registry`, password from `Secret/-app`), and give +Keycloak its own database and role. Then no workload holds superuser rights. + +## Secrets + +Three values, none of which this chart ever renders: + +| Value | Setting | +|---|---| +| Database password | `database.passwordSecret` | +| `admin-api` client secret | `keycloak.adminClientSecret` | +| Default password for Keycloak users the registry creates | `defaultUserPasswordSecret` | + +This chart renders none of them — it only references Secrets, which are created +by whatever manages secrets in that environment. For dev that is a +`kubectl create secret`; the example values show the exact command. + +## Probes + +`/health`, matching the compose healthcheck. Sunbird's own chart probes +`/api/docs/swagger.json`, which additionally requires swagger to be enabled; +`/health` is the narrower check. The registry migrates its schema on first start, +so `startupProbe` carries the slow path. + +## Configuration + +See [`values.yaml`](./values.yaml) for the full commented schema and +[`examples/registry.dev.yaml`](./examples/registry.dev.yaml) for a +per-environment file. + +The `*_enabled: "false"` flags in `envConfig` are deliberate. Each one that is +turned on pulls in another Sunbird RC service that is not deployed — the registry +will start, then fail at the first request that needs it. + +## Versioning + +Every change needs a `version` bump in `Chart.yaml` and an entry in +[`CHANGELOG.md`](./CHANGELOG.md) — see [`CONVENTIONS.md`](../../CONVENTIONS.md). diff --git a/charts/registry/ci/lint-values.yaml b/charts/registry/ci/lint-values.yaml new file mode 100644 index 0000000..dbdaf7b --- /dev/null +++ b/charts/registry/ci/lint-values.yaml @@ -0,0 +1,18 @@ +# Minimum values that let `helm template` run in CI. Dummy references only - +# see examples/ for real per-environment configuration. +database: + host: registry-db-rw + passwordSecret: + name: registry-db-app +keycloak: + url: http://keycloak:8080/auth + adminClientSecret: + name: registry-keycloak +defaultUserPasswordSecret: + name: registry-keycloak + +# image.repository is intentionally empty in the chart, so CI supplies one. +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-core + tag: "v2.0.0" diff --git a/charts/registry/examples/registry.dev.yaml b/charts/registry/examples/registry.dev.yaml new file mode 100644 index 0000000..94fa2aa --- /dev/null +++ b/charts/registry/examples/registry.dev.yaml @@ -0,0 +1,84 @@ +# Example: the OAN participant registry, dev environment. +# +# helm install registry charts/registry \ +# -n oan-registry -f charts/registry/examples/registry.dev.yaml +# +# Install this LAST - it needs the database, and it needs the regenerated +# admin-api client secret from Keycloak. See README.md for the full order. + +# No fullnameOverride needed: the chart is named "registry", so a release named +# "registry" already produces Service/registry. + +# The image the compose stack runs, so dev deploys what was verified locally. +# The chart default is empty on purpose - see values.yaml. +image: + registry: ghcr.io + repository: sunbird-rc/sunbird-rc-core + tag: "v2.0.0" + +# Connects as the OWNER of its own database, not as the superuser. Compose uses +# `postgres` because everything shares one database there; in the cluster the +# registry owns `registry` and Keycloak owns `keycloak`, so neither needs +# superuser rights. +database: + host: registry-db-rw + port: 5432 + name: registry + user: registry + # The Secret the cluster's bootstrap owner was created from. + passwordSecret: + name: registry-db-app + key: password + +keycloak: + # Must match the keycloak release's in-cluster DNS, including /auth, and must + # equal the issuer Keycloak puts in its tokens. + url: http://keycloak.oan-registry.svc.cluster.local:8080/auth + realm: sunbird-rc + adminClientId: admin-api + clientId: registry-frontend + # The REGENERATED admin-api secret. Until ESO is available: + # kubectl -n oan-registry create secret generic registry-keycloak \ + # --from-literal=keycloakAdminClientSecret='' \ + # --from-literal=registryDefaultUserPassword='' + adminClientSecret: + name: registry-keycloak + key: keycloakAdminClientSecret + +authenticationEnabled: true + +keycloakUserSetPassword: true +defaultUserPasswordSecret: + name: registry-keycloak + key: registryDefaultUserPassword + +# Ships files/schemas/*.json (currently just Participant.json) +schemas: + existingConfigMap: "" + +ingress: + enabled: false + +resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +commonLabels: + oan.in/environment: dev + +# Blocks startup until the database is reachable AND Keycloak's realm endpoint +# answers. The Keycloak check is HTTP against the realm, not TCP against the +# port: the realm is imported during Keycloak's first start, so the port opens +# well before the realm the registry needs exists. +waitFor: + enabled: true + database: true + keycloak: true + +# `helm test registry -n oan-registry` checks /health. +tests: + enabled: true diff --git a/charts/registry/examples/registry.prod.yaml b/charts/registry/examples/registry.prod.yaml new file mode 100644 index 0000000..1a56e3b --- /dev/null +++ b/charts/registry/examples/registry.prod.yaml @@ -0,0 +1,144 @@ +# Example: the OAN participant registry, production. +# +# helm install registry charts/registry \ +# -n oan-registry -f charts/registry/examples/registry.prod.yaml +# +# The differences from dev are all here, per issue #30's requirement that +# environment differences live only in values files. + +# --------------------------------------------------------------------------- +# TODO: set the production image. Left empty on purpose - the image OAN will +# actually deploy is not decided yet (upstream ghcr, a mirror in our own +# registry, or an OAN-built image), and guessing here would be a guess that +# looks like a decision. +# +# The chart FAILS TO RENDER while repository is empty, which is deliberate: an +# empty repository would otherwise produce "ghcr.io/:v2.0.0" and only fail later +# as an ImagePullBackOff. So this example is not installable until it is filled +# in. +# +# When filling it in, prefer a digest over a tag. A tag can be repointed at +# different bits; a digest cannot, which is what makes a rollback land on the +# same image it did before. +# +# image: +# registry: +# repository: +# tag: "v2.0.0" +# digest: "sha256:..." +# --------------------------------------------------------------------------- +image: + registry: "" + repository: "" + tag: "" + digest: "" + +# Two replicas minimum: one can be drained without an outage, and the PDB below +# needs somewhere to move traffic. +replicaCount: 2 + +database: + host: registry-db-rw + port: 5432 + name: registry + # Production connects as the database OWNER, not the superuser. This is the + # main security difference from dev: the registry has no rights beyond its own + # database. Requires the cluster's bootstrap owner to be `registry`. + user: registry + passwordSecret: + name: registry-db-app + key: password + +keycloak: + url: http://keycloak.oan-registry.svc.cluster.local:8080/auth + realm: sunbird-rc + adminClientId: admin-api + clientId: registry-frontend + adminClientSecret: + name: registry-keycloak + key: keycloakAdminClientSecret + +authenticationEnabled: true + +keycloakUserSetPassword: true +defaultUserPasswordSecret: + name: registry-keycloak + key: registryDefaultUserPassword + +# Secrets are created out of band; this chart renders none of them. The Secret +# named above must carry both keys: +# kubectl -n oan-registry create secret generic registry-keycloak \ +# --from-literal=keycloakAdminClientSecret='' \ +# --from-literal=registryDefaultUserPassword='' + +resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi + +# Survives a single node drain. +podDisruptionBudget: + enabled: true + minAvailable: 1 + +# Off until there is load data to size it from. Note every replica opens its own +# connection pool, so check the cluster's max_connections before enabling. +autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 4 + targetCPUUtilizationPercentage: 70 + +# Spread replicas across nodes, so one node failing does not take both. +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: registry + app.kubernetes.io/instance: registry + +# Hardened. Verify the image runs as non-root before enabling in a new +# environment - a mismatch shows up as a permission error at startup. +podSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + enabled: true + allowPrivilegeEscalation: false + runAsNonRoot: true + # Left false: the JVM writes to /tmp. Enabling it needs an emptyDir mounted + # there via extraVolumes, and verification that nothing else writes to disk. + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + +ingress: + enabled: false + # When the AWS Load Balancer Controller is installed: + # className: alb + # annotations: + # alb.ingress.kubernetes.io/scheme: internal + # alb.ingress.kubernetes.io/target-type: ip + # hosts: + # - host: registry.oan.example + # paths: + # - path: / + # pathType: Prefix + +envConfig: + logging.level.root: WARN + +commonLabels: + oan.in/environment: prod diff --git a/charts/registry/files/schemas/Participant.json b/charts/registry/files/schemas/Participant.json new file mode 100644 index 0000000..748beb7 --- /dev/null +++ b/charts/registry/files/schemas/Participant.json @@ -0,0 +1,170 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "Participant": { + "$ref": "#/definitions/Participant" + } + }, + "required": [ + "Participant" + ], + "title": "Participant", + "definitions": { + "Participant": { + "$id": "#/properties/Participant", + "type": "object", + "title": "OAN network participant", + "comment": "Flat by design: nested objects would each become their own graph node with their own osid. Every field here is a scalar or an array of scalars, so one participant is one record with one osid.", + "required": [ + "participant_id", + "display_name", + "roles", + "status", + "record_version", + "updated_at", + "signing_public_key", + "endpoint_url", + "domain" + ], + "additionalProperties": false, + "properties": { + "participant_id": { + "type": "string", + "title": "Business identifier of the participant, unique across the network" + }, + "display_name": { + "type": "string", + "title": "Human-readable name of the participant" + }, + "roles": { + "type": "array", + "title": "Roles this participant holds on the network", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "network_operator", + "provider", + "consumer" + ] + } + }, + "status": { + "type": "string", + "title": "Lifecycle status of the participant", + "enum": [ + "active", + "inactive" + ] + }, + "domain": { + "type": "string", + "title": "Kind of participant, e.g. weather, market, credit, advisory" + }, + "record_version": { + "type": "integer", + "title": "Version counter of this record, starts at 1", + "minimum": 1 + }, + "updated_at": { + "type": "string", + "format": "date-time", + "title": "ISO-8601 timestamp of the last update" + }, + "endpoint_url": { + "type": "string", + "title": "Base URL where requests to this participant are sent once resolved" + }, + "endpoint_type": { + "type": "string", + "title": "What this endpoint serves", + "enum": [ + "onix", + "callback", + "api" + ] + }, + "signing_public_key": { + "type": "string", + "title": "Base64-encoded public key a receiver uses to verify this participant's signed requests" + }, + "signing_algorithm": { + "type": "string", + "title": "Algorithm of the signing key pair", + "enum": [ + "ed25519" + ] + }, + "key_valid_from": { + "type": "string", + "format": "date-time", + "title": "ISO-8601 timestamp from which the signing key is usable" + }, + "key_valid_until": { + "type": "string", + "format": "date-time", + "title": "ISO-8601 timestamp after which the signing key must not be trusted" + }, + "osid": { + "type": "string", + "title": "System id, assigned by the registry - accepted on update, never set by a client" + }, + "osOwner": { + "type": "array", + "items": { + "type": "string" + }, + "title": "System field: keycloak subject(s) that own this record" + }, + "osCreatedAt": { + "type": "string", + "title": "System field: creation timestamp" + }, + "osUpdatedAt": { + "type": "string", + "title": "System field: last update timestamp" + }, + "osCreatedBy": { + "type": "string", + "title": "System field: creator" + }, + "osUpdatedBy": { + "type": "string", + "title": "System field: last updater" + }, + "_status": { + "type": [ + "boolean", + "string", + "null" + ], + "title": "System field: storage-level soft-delete flag" + } + } + } + }, + "_osConfig": { + "systemFields": [ + "osCreatedAt", + "osUpdatedAt", + "osCreatedBy", + "osUpdatedBy" + ], + "uniqueIndexFields": [ + "participant_id" + ], + "indexFields": [ + "status", + "domain" + ], + "roles": [ + "admin", + "network_operator" + ], + "inviteRoles": [ + "admin", + "network_operator" + ] + } +} \ No newline at end of file diff --git a/charts/registry/templates/NOTES.txt b/charts/registry/templates/NOTES.txt new file mode 100644 index 0000000..6d6eee5 --- /dev/null +++ b/charts/registry/templates/NOTES.txt @@ -0,0 +1,35 @@ +{{ .Chart.Name }} installed as release "{{ .Release.Name }}". + +Resources in namespace "{{ .Release.Namespace }}": + - Deployment/{{ include "registry.fullname" . }} ({{ .Values.replicaCount }} replica(s)) + - Service/{{ include "registry.fullname" . }} ({{ .Values.service.type }} on port {{ .Values.service.port }}) + - ConfigMap/{{ include "registry.envConfigMapName" . }} +{{- if include "registry.renderSchemasConfigMap" . }} + - ConfigMap/{{ include "registry.schemasConfigMapName" . }} (entity schemas) +{{- end }} +{{- if .Values.serviceAccount.enabled }} + - ServiceAccount/{{ include "registry.serviceAccountName" . }} +{{- end }} +{{- if .Values.ingress.enabled }} + - Ingress/{{ include "registry.fullname" . }} +{{- end }} + +Wired to: + database: {{ include "registry.jdbcUrl" . }} (as {{ .Values.database.user }}) + keycloak: {{ .Values.keycloak.url }} (realm {{ .Values.keycloak.realm }}) + auth: {{ if .Values.authenticationEnabled }}enabled - role checks apply{{ else }}DISABLED - no role checks{{ end }} + +In-cluster endpoint: + http://{{ include "registry.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} + +Check it: + kubectl -n {{ .Release.Namespace }} rollout status deployment/{{ include "registry.fullname" . }} + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "registry.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl http://localhost:{{ .Values.service.port }}/health + +If authenticated calls come back 401 or 403, check these three in order - they +are the usual causes, and none of them look like an auth problem in the logs: + 1. keycloak.url must match the issuer Keycloak puts in its tokens, exactly. + 2. keycloak.adminClientSecret must be the REGENERATED admin-api secret; the + one in the realm export is masked and never works. + 3. The realm ({{ .Values.keycloak.realm }}) must actually be imported. diff --git a/charts/registry/templates/_helpers.tpl b/charts/registry/templates/_helpers.tpl new file mode 100644 index 0000000..d2df577 --- /dev/null +++ b/charts/registry/templates/_helpers.tpl @@ -0,0 +1,181 @@ +{{/* +# ============================================================================ +# REGISTRY (SUNBIRD RC) CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the database, +# Keycloak and schema wiring the registry needs. +# ============================================================================ +*/}} + +{{- define "registry.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "registry.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "registry.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "registry.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "registry.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "registry.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{- define "registry.envConfigMapName" -}} +{{- include "oan-common.envConfigMapName" . -}} +{{- end }} + +{{/* +JDBC URL for the registry database. +*/}} +{{- define "registry.jdbcUrl" -}} +{{- $db := .Values.database -}} +{{- printf "jdbc:postgresql://%s:%v/%s" $db.host $db.port $db.name -}} +{{- end }} + +{{/* +Name of the ConfigMap holding the entity schemas. An existing ConfigMap wins, +so schemas can be managed outside this chart. +*/}} +{{- define "registry.schemasConfigMapName" -}} +{{- if .Values.schemas.existingConfigMap -}} +{{- .Values.schemas.existingConfigMap -}} +{{- else -}} +{{- printf "%s-schemas" (include "registry.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Emits "true" when this chart renders the schemas ConfigMap itself. +*/}} +{{- define "registry.renderSchemasConfigMap" -}} +{{- if not .Values.schemas.existingConfigMap -}} +{{- true -}} +{{- end -}} +{{- end }} + +{{/* +The schema files shipped with the chart, as ConfigMap data entries. The registry +reads every .json under its schema directory as an entity definition, so an +empty set means a registry that knows about no entities at all. +*/}} +{{- define "registry.schemasData" -}} +{{- $files := .Files.Glob .Values.schemas.filesGlob -}} +{{- if and (not $files) (not .Values.schemas.inline) -}} +{{- fail (printf "%s: no schema files matched %q in the chart and schemas.inline is empty. The registry needs at least one entity definition." .Chart.Name .Values.schemas.filesGlob) -}} +{{- end -}} +{{- range $path, $_ := $files }} +{{ base $path }}: |- + {{- $.Files.Get $path | nindent 2 }} +{{- end }} +{{- range $name, $content := .Values.schemas.inline }} +{{ $name }}: |- + {{- $content | nindent 2 }} +{{- end }} +{{- end }} + +{{/* +Environment the registry needs beyond the flags in envConfig: everything derived +from the database and Keycloak settings, plus the three secret values. + +Mirrors registry/docker-compose.yml. `sunbird_sso_url` and OAUTH2_RESOURCES_0_URI +must agree with the issuer Keycloak actually puts in its tokens, or every request +is rejected as unauthorised. +*/}} +{{- define "registry.env" -}} +{{- $db := .Values.database -}} +{{- $kc := .Values.keycloak -}} +{{- if not $db.host }} +{{- fail (printf "%s: database.host is required - point it at the PostgreSQL primary service, e.g. registry-db-rw" .Chart.Name) }} +{{- end }} +{{- if not $db.passwordSecret.name }} +{{- fail (printf "%s: database.passwordSecret.name is required - this chart renders no passwords" .Chart.Name) }} +{{- end }} +{{- if not $kc.url }} +{{- fail (printf "%s: keycloak.url is required, including the /auth context path, e.g. http://keycloak:8080/auth" .Chart.Name) }} +{{- end }} +{{- if not $kc.adminClientSecret.name }} +{{- fail (printf "%s: keycloak.adminClientSecret.name is required. Regenerate the admin-api client secret in the Keycloak console after the realm import, then store it." .Chart.Name) }} +{{- end }} +{{- if and .Values.keycloakUserSetPassword (not .Values.defaultUserPasswordSecret.name) }} +{{- fail (printf "%s: keycloakUserSetPassword is true, so defaultUserPasswordSecret.name is required - it is the password the registry sets on Keycloak users it creates" .Chart.Name) }} +{{- end }} +{{- $kcUrl := $kc.url | trimSuffix "/" }} +- name: connectionInfo_uri + value: {{ include "registry.jdbcUrl" . | quote }} +- name: connectionInfo_username + value: {{ $db.user | quote }} +- name: connectionInfo_password + valueFrom: + secretKeyRef: + name: {{ $db.passwordSecret.name }} + key: {{ $db.passwordSecret.key }} +- name: authentication_enabled + value: {{ .Values.authenticationEnabled | quote }} +- name: sunbird_sso_realm + value: {{ $kc.realm | quote }} +- name: sunbird_sso_url + value: {{ $kcUrl | quote }} +- name: OAUTH2_RESOURCES_0_URI + value: {{ printf "%s/realms/%s" $kcUrl $kc.realm | quote }} +- name: OAUTH2_RESOURCES_0_PROPERTIES_ROLES_PATH + value: {{ $kc.rolesPath | quote }} +- name: identity_provider + value: {{ $kc.identityProvider | quote }} +- name: sunbird_sso_admin_client_id + value: {{ $kc.adminClientId | quote }} +- name: sunbird_sso_client_id + value: {{ $kc.clientId | quote }} +- name: sunbird_sso_admin_client_secret + valueFrom: + secretKeyRef: + name: {{ $kc.adminClientSecret.name }} + key: {{ $kc.adminClientSecret.key }} +- name: sunbird_keycloak_user_set_password + value: {{ .Values.keycloakUserSetPassword | quote }} +{{- with .Values.defaultUserPasswordSecret.name }} +- name: sunbird_keycloak_user_password + valueFrom: + secretKeyRef: + name: {{ . }} + key: {{ $.Values.defaultUserPasswordSecret.key }} +{{- end }} +{{- with (include "oan-common.env" . | trim) }} +{{ . }} +{{- end }} +{{- end }} + +{{/* +Dependency waits, derived from this chart's own settings so there is nothing to +keep in sync. + +The Keycloak check is HTTP against the realm endpoint, not TCP against the port: +the realm is imported during Keycloak's first start, so the port opens well +before the realm the registry needs actually exists. A TCP check would pass too +early and the registry would start against a Keycloak that cannot yet issue it a +usable token. +*/}} +{{- define "registry.waitFor" -}} +{{- $tcp := list -}} +{{- $http := list -}} +{{- if .Values.waitFor.database }} +{{- $tcp = append $tcp (dict "name" "database" "host" .Values.database.host "port" .Values.database.port) -}} +{{- end -}} +{{- if .Values.waitFor.keycloak }} +{{- $url := printf "%s/realms/%s" (.Values.keycloak.url | trimSuffix "/") .Values.keycloak.realm -}} +{{- $http = append $http (dict "name" "keycloak" "url" $url) -}} +{{- end -}} +{{- $tcp = concat $tcp (.Values.waitFor.extraTcp | default list) -}} +{{- $http = concat $http (.Values.waitFor.extraHttp | default list) -}} +{{- include "oan-common.waitFor" (dict "ctx" . "tcp" $tcp "http" $http) -}} +{{- end }} diff --git a/charts/registry/templates/configmap-schemas.yaml b/charts/registry/templates/configmap-schemas.yaml new file mode 100644 index 0000000..d2ed8c8 --- /dev/null +++ b/charts/registry/templates/configmap-schemas.yaml @@ -0,0 +1,18 @@ +{{- if include "registry.renderSchemasConfigMap" . }} +{{/* +Entity definitions, mounted at schemas.mountPath. The registry treats every +.json here as an entity it serves APIs for. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "registry.schemasConfigMapName" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "registry.schemasData" . | trim | nindent 2 }} +{{- end }} diff --git a/charts/registry/templates/configmap.yaml b/charts/registry/templates/configmap.yaml new file mode 100644 index 0000000..aa8ade7 --- /dev/null +++ b/charts/registry/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "registry.envConfigMapName" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- with (include "oan-common.envConfigMapData" . | trim) }} +data: + {{- . | nindent 2 }} +{{- else }} +data: {} +{{- end }} diff --git a/charts/registry/templates/deployment.yaml b/charts/registry/templates/deployment.yaml new file mode 100644 index 0000000..c03f6f5 --- /dev/null +++ b/charts/registry/templates/deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "registry.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "registry.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/env-config: {{ include "oan-common.checksumAnnotation" . }} + checksum/schemas: {{ include "registry.schemasData" . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "registry.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + {{- $init := include "registry.waitFor" . | trim }} + {{- $extraInit := "" }} + {{- with .Values.extraInitContainers }} + {{- $extraInit = toYaml . | trim }} + {{- end }} + {{- if or $init $extraInit }} + initContainers: + {{- with $init }} + {{- . | nindent 8 }} + {{- end }} + {{- with $extraInit }} + {{- . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "registry.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{- include "registry.env" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "registry.envConfigMapName" . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + - name: schemas + mountPath: {{ .Values.schemas.mountPath }} + readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: schemas + configMap: + name: {{ include "registry.schemasConfigMapName" . }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/registry/templates/hpa.yaml b/charts/registry/templates/hpa.yaml new file mode 100644 index 0000000..4157d8d --- /dev/null +++ b/charts/registry/templates/hpa.yaml @@ -0,0 +1,51 @@ +{{- if .Values.autoscaling.enabled }} +{{/* +The registry is stateless - it holds no session state and writes everything to +PostgreSQL - so it scales horizontally. Two things to keep in mind: + + - Every replica opens its own connection pool, so scaling out multiplies + connections against the database. Check `max_connections` on the cluster + before raising maxReplicas much. + - Autoscaling takes over `replicas`, so replicaCount is ignored once this is + enabled. The Deployment deliberately omits `replicas` in that case, or the + two would fight on every reconcile. +*/}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "registry.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- with .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/registry/templates/ingress.yaml b/charts/registry/templates/ingress.yaml new file mode 100644 index 0000000..285444a --- /dev/null +++ b/charts/registry/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- range . }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "registry.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/registry/templates/poddisruptionbudget.yaml b/charts/registry/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..d9ae912 --- /dev/null +++ b/charts/registry/templates/poddisruptionbudget.yaml @@ -0,0 +1,33 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Caps how many pods can be voluntarily disrupted at once - node drains, cluster +upgrades, autoscaler scale-down. + +With replicaCount: 1 a PDB of minAvailable: 1 blocks drains entirely, because +evicting the only pod would breach it. That is why this is disabled by default +and should be enabled together with replicaCount >= 2. +*/}} +{{- if and (eq (int .Values.replicaCount) 1) (not .Values.podDisruptionBudget.allowSingleReplica) }} +{{- fail (printf "%s: podDisruptionBudget is enabled with replicaCount 1, which blocks node drains entirely. Raise replicaCount to 2+, or set podDisruptionBudget.allowSingleReplica=true if you accept that." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "registry.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/registry/templates/service.yaml b/charts/registry/templates/service.yaml new file mode 100644 index 0000000..13ccd91 --- /dev/null +++ b/charts/registry/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "registry.fullname" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "registry.selectorLabels" . | nindent 4 }} diff --git a/charts/registry/templates/serviceaccount.yaml b/charts/registry/templates/serviceaccount.yaml new file mode 100644 index 0000000..b80401c --- /dev/null +++ b/charts/registry/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if include "oan-common.serviceAccount.enabled" . }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "registry.serviceAccountName" . }} + labels: + {{- include "registry.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/registry/templates/tests/test-connection.yaml b/charts/registry/templates/tests/test-connection.yaml new file mode 100644 index 0000000..b6ad6e0 --- /dev/null +++ b/charts/registry/templates/tests/test-connection.yaml @@ -0,0 +1,36 @@ +{{- if .Values.tests.enabled }} +{{/* +`helm test` check: the registry answers on /health. + +Deliberately does not test an authenticated endpoint: that needs a real token, +and a failure there would be ambiguous between "registry is broken" and +"Keycloak wiring is wrong". Health tells you the pod is serving; the NOTES +explain how to check the auth chain. +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "registry.fullname" . }}-test-connection + labels: + {{- include "registry.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: test + image: {{ printf "%s/%s:%s" .Values.tests.image.registry .Values.tests.image.repository .Values.tests.image.tag | quote }} + command: + - /bin/sh + - -c + - | + set -e + url="http://{{ include "registry.fullname" . }}:{{ .Values.service.port }}/health" + echo "GET $url" + wget -q -O- --timeout=10 "$url" + echo "" + echo "OK: registry is serving /health" + resources: + {{- toYaml .Values.tests.resources | nindent 8 }} +{{- end }} diff --git a/charts/registry/values.yaml b/charts/registry/values.yaml new file mode 100644 index 0000000..f85b7cd --- /dev/null +++ b/charts/registry/values.yaml @@ -0,0 +1,353 @@ +# ============================================================================ +# registry - default values +# +# The OAN participant registry, on Sunbird RC core. Defaults mirror the verified +# compose stack (registry/docker-compose.yml): participant records and lookup +# only, with every optional Sunbird RC subsystem (credentials, DIDs, +# certificates, file storage, notifications, webhooks) switched off. +# +# Depends on two things already running: +# - PostgreSQL -> database.host +# - Keycloak -> keycloak.url +# +# Minimum you must set: +# - database.host, database.passwordSecret.name +# - keycloak.url, keycloak.adminClientSecret.name +# - defaultUserPasswordSecret.name +# ============================================================================ + +replicaCount: 1 + +# --------------------------------------------------------------------------- +# Image +# +# repository is intentionally EMPTY: the image OAN deploys is not settled yet, and +# a default here would look like a decision that has not been made. The render +# fails while it is empty rather than producing "ghcr.io/:v2.0.0", which would +# only surface later as an ImagePullBackOff. +# +# The compose stack runs ghcr.io/sunbird-rc/sunbird-rc-core:v2.0.0, which is what +# the dev example values use. Set this per environment. +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: "" + # RELEASE_VERSION in the compose stack + tag: "v2.0.0" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: true + +service: + type: ClusterIP + port: 8081 + targetPort: 8081 + annotations: {} + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: registry.local + paths: + - path: / + pathType: Prefix + tls: [] + +# --------------------------------------------------------------------------- +# Database +# +# Compose runs the registry as the postgres superuser against the same database +# Keycloak uses. On a cluster, prefer a dedicated owner role: set user to the +# CNPG cluster's bootstrap owner and give Keycloak its own database. +# --------------------------------------------------------------------------- +database: + # PostgreSQL primary service, e.g. "registry-db-rw" + host: "" + port: 5432 + name: registry + user: postgres + # Secret holding the database password. Required. + passwordSecret: + name: "" + key: password + +# --------------------------------------------------------------------------- +# Keycloak +# +# `url` must include the /auth context path the legacy Keycloak serves under, +# and must be the URL Keycloak itself puts in the `iss` claim. If they disagree, +# every authenticated request is rejected - and the failure looks like a +# permissions problem, not a configuration one. +# +# The keycloak chart prints the exact value to use in its NOTES. +# --------------------------------------------------------------------------- +keycloak: + # e.g. http://keycloak.oan-registry.svc.cluster.local:8080/auth + url: "" + realm: sunbird-rc + # Confidential client the registry uses for admin operations on Keycloak + adminClientId: admin-api + # Client tokens are issued to + clientId: registry-frontend + rolesPath: realm_access.roles + identityProvider: dev.sunbirdrc.auth.keycloak.KeycloakProviderImpl + # Secret holding the admin-api client secret. + # + # The realm export ships this masked, so it is NOT usable until someone + # regenerates it: Keycloak console -> realm sunbird-rc -> Clients -> admin-api + # -> Credentials -> Regenerate Secret. First install is two-phase because of + # this. + adminClientSecret: + name: "" + key: keycloakAdminClientSecret + +# Role checks on Participant only apply while this is true. +authenticationEnabled: true + +# Whether the registry sets a password on Keycloak users it creates. +keycloakUserSetPassword: true +# The password it sets. Required while keycloakUserSetPassword is true. +defaultUserPasswordSecret: + name: "" + key: registryDefaultUserPassword + +# --------------------------------------------------------------------------- +# Entity schemas +# +# Every .json under the registry's schema directory is an entity definition. +# The chart ships the participant schema and mounts it at mountPath. +# +# NOTE: this schema was built and verified against the local stack, but per +# engineering-tracker #33 it has no design issue of its own yet. Expect it to +# change - a schema change is a chart release. +# --------------------------------------------------------------------------- +schemas: + filesGlob: "files/schemas/*.json" + mountPath: /home/sunbirdrc/config/public/_schemas + # Manage schemas outside this chart instead of shipping them + existingConfigMap: "" + # Extra or overriding schemas, as filename -> JSON string + inline: {} + +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +# --------------------------------------------------------------------------- +# Probes +# +# /health is what the compose healthcheck uses. Sunbird's own Helm chart probes +# /api/docs/swagger.json instead, which additionally requires swagger to be +# enabled; /health is the narrower, more honest check. +# +# The registry runs schema migrations on first start, so startupProbe carries +# the slow path. +# --------------------------------------------------------------------------- +startupProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 30 + +livenessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 0 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +readinessProbe: + enabled: true + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + +podSecurityContext: + enabled: false + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + +securityContext: + enabled: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} + +# --------------------------------------------------------------------------- +# Dependency waiting +# +# Kubernetes has no equivalent of compose's `depends_on: condition: +# service_healthy`. Without these init containers the pod starts before its +# dependencies are up, fails, and crashloops with backoff - which recovers on +# its own but makes a first install look broken. +# --------------------------------------------------------------------------- +waitFor: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + timeoutSeconds: 300 + intervalSeconds: 3 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + # Wait for the database. Host and port come from `database` above. + database: true + # Wait for Keycloak's REALM endpoint, derived from keycloak.url and + # keycloak.realm - not just its port. The realm is imported during Keycloak's + # first start, so the port opens well before the realm exists, and a TCP check + # would let the registry start against a Keycloak that cannot yet issue it a + # usable token. + keycloak: true + extraTcp: [] + extraHttp: [] + +# Extra init containers, appended after the waitFor ones. Rendered verbatim. +extraInitContainers: [] + +# Extra volumes and mounts, rendered verbatim. For certificates, extra config, +# or a writable scratch directory when readOnlyRootFilesystem is enabled. +extraVolumes: [] +extraVolumeMounts: [] + +# --------------------------------------------------------------------------- +# PodDisruptionBudget +# +# Caps voluntary disruptions - node drains, cluster upgrades, autoscaler +# scale-down. Disabled by default because with a single replica a PDB of +# minAvailable: 1 blocks drains entirely; the render fails if you enable it +# anyway without acknowledging that. +# --------------------------------------------------------------------------- +podDisruptionBudget: + enabled: false + minAvailable: 1 + # Set one or the other, not both. + maxUnavailable: "" + # Enable a PDB with a single replica anyway, accepting that node drains block. + allowSingleReplica: false + +# --------------------------------------------------------------------------- +# `helm test` checks +# +# helm test -n +# +# Runs after install to confirm the service actually answers, which is the +# difference between "the pod is running" and "the deployment works". +# --------------------------------------------------------------------------- +tests: + enabled: true + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi + +# --------------------------------------------------------------------------- +# Autoscaling +# +# The registry is stateless, so it scales horizontally. Note that every replica +# opens its own connection pool, so check the database's max_connections before +# raising maxReplicas much. When enabled, replicaCount is ignored. +# --------------------------------------------------------------------------- +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + # Set to a number to also scale on memory. + targetMemoryUtilizationPercentage: "" + # Scale-up/scale-down tuning, rendered verbatim into spec.behavior. + behavior: {} + + +# --------------------------------------------------------------------------- +# ENVIRONMENT CONFIGURATION +# +# Rendered into a ConfigMap and injected with envFrom. These are exactly the +# non-secret, non-derived variables from the compose stack. Database and +# Keycloak variables are derived from the sections above instead, so they are +# not repeated here. +# +# The *_enabled flags are all false on purpose: this deployment is participant +# records and lookup only. Turning any of them on pulls in another Sunbird RC +# service that is not deployed - the registry will start and then fail at the +# first request that needs it. +# +# NEVER put secrets here. +# --------------------------------------------------------------------------- +envConfig: + search_providerName: dev.sunbirdrc.registry.service.NativeSearchService + manager_type: DefinitionsManager + expand_reference: "false" + encryption_enabled: "false" + event_enabled: "false" + idgen_enabled: "false" + claims_enabled: "false" + did_enabled: "false" + signature_enabled: "false" + certificate_enabled: "false" + filestorage_enabled: "false" + notification_enabled: "false" + notification_async_enabled: "false" + async_enabled: "false" + webhook_enabled: "false" + registry_base_apis_enable: "false" + swagger_title: OAN Registry + logging.level.root: INFO + +# Extra env vars from specific Secret keys, beyond the three wired above. +secretEnv: {} +extraEnv: [] +envFromSecrets: [] + +commonLabels: {} +commonAnnotations: {} From b353a7515d36bbf8c38dfff3f636a5d9ce3d7db7 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:35:27 +0530 Subject: [PATCH 54/81] docs: list the seven charts and their install order in README [#53] Records the chart inventory and the order releases have to go out in, since the database cluster has to exist before the migration Job and the services that connect to it. --- README.md | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 81364f6..ed05c3c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,146 @@ # helmcharts -Helm charts for deploying and managing platform services + +Helm charts for deploying and managing OpenAgriNet (OAN) platform services. + +## Charts + +| Chart | Type | Purpose | +|---|---|---| +| [`oan-common`](charts/oan-common) | library | Shared template helpers — names, labels, image refs, probes, resources, service account, env config, dependency waits. Renders nothing; never installed directly. | +| [`oan-template`](charts/oan-template) | application | Complete, working starter chart built on `oan-common`. Copy it to bootstrap a service chart. | +| [`postgresql-cnpg`](charts/postgresql-cnpg) | application | CloudNativePG-managed PostgreSQL cluster. One release per database. Requires the CNPG operator. | +| [`postgresql-migration`](charts/postgresql-migration) | application | Flyway migrations as a Job. Creates the per-service databases and applies versioned SQL. | +| [`keycloak`](charts/keycloak) | application | Auth for the registry, on the Sunbird RC Keycloak image. Imports the realm the registry expects. | +| [`registry`](charts/registry) | application | The OAN participant registry, on Sunbird RC core. Needs `postgresql-cnpg` and `keycloak`. | +| [`discovery`](charts/discovery) | application | The OAN Beckn discover-and-publish service. Needs `postgresql-cnpg` **with pgvector**. | + +## How they fit together + +``` +charts/ +├── oan-common/ # library chart — shared helpers +├── oan-template/ # starter chart — copy this to build a service chart +├── postgresql-cnpg/ # data store +├── postgresql-migration/# schema migrations (Flyway Job) +├── keycloak/ # auth for the registry +├── registry/ # the participant registry +└── discovery/ # the Beckn discover-and-publish service +``` + +Every chart depends on `oan-common` via `file://../oan-common`. + +## The registry stack + +Three charts, deployed in this order — the ordering is not optional: + +```bash +# 1. Database cluster. Creates BOTH databases: `registry` via bootstrap.initdb +# and `keycloak` via a CNPG Database object, each owned by its own role. +helm install registry-db charts/postgresql-cnpg -n oan-registry -f charts/postgresql-cnpg/examples/registry-db.dev.yaml +# 2. Keycloak — imports the sunbird-rc realm it ships with, on first start +helm install keycloak charts/keycloak -n oan-registry -f charts/keycloak/examples/keycloak.dev.yaml +# MANUAL STEP: regenerate the admin-api client secret in the Keycloak console — +# the realm export ships it masked, so the registry cannot authenticate without this. +# 3. Registry +helm install registry charts/registry -n oan-registry -f charts/registry/examples/registry.dev.yaml +``` + +`postgresql-migration` is deliberately not in that list: both databases come from +the cluster chart, and Sunbird RC and Keycloak each manage their own schema, so +there is nothing for Flyway to apply yet. It joins the flow when OAN adds schemas +of its own. + +Their configuration is ported from the verified `registry/docker-compose.yml` +stack at exact environment-variable parity (32 for the registry, 10 for +Keycloak), so a cluster deploy reproduces what was tested locally. Each chart's +README documents where it deliberately deviates and why. Full walkthrough: +[`charts/registry/README.md`](charts/registry/README.md). + +## The discovery service + +Two charts, and one prerequisite that is easy to miss: + +```bash +# 1. Its own database — pgvector on PostgreSQL 16, with the `vector` extension +# created at bootstrap. Both are required: no stock CNPG operand image has +# pgvector, and `vector` is not a trusted extension, so the owner the service +# connects as cannot create it itself. +helm install discovery-db charts/postgresql-cnpg -n oan-discovery -f charts/postgresql-cnpg/examples/discovery-db.dev.yaml +# 2. The service +helm install discovery charts/discovery -n oan-discovery -f charts/discovery/examples/discovery.dev.yaml +``` + +It shares no database and no Keycloak with the registry stack, so the two are +independent installs. Full walkthrough, including how the DSN and the Beckn +specification are supplied: +[`charts/discovery/README.md`](charts/discovery/README.md). + +Service charts depend on `oan-common` and call its helpers through thin +chart-local wrappers. That keeps naming, labelling, probe, resource, and secret +conventions identical across every OAN chart, and means a convention change is +one edit in the library rather than one edit per chart. + +## Quick start + +Build a service chart from the template: + +```bash +# 1. Copy the starter chart +cp -r charts/oan-template charts/oan-my-service + +# 2. In charts/oan-my-service/Chart.yaml set name: oan-my-service +# and appVersion to the image tag you deploy by default. +# Keep the oan-common dependency. + +# 3. Rename the chart-local helpers to your service name. Change only the left +# side of each define in templates/_helpers.tpl (oan-template.* -> +# oan-my-service.*); the oan-common.* include inside the body stays. This +# renames the defines and the include calls together: +grep -rl 'oan-template\.' charts/oan-my-service | xargs sed -i '' 's/oan-template\./oan-my-service./g' +# (sed -i '' is the macOS form; on Linux use sed -i) + +# 4. Set image, ports, probe paths, resources and envConfig in +# charts/oan-my-service/values.yaml + +# 5. Validate +./scripts/lint-charts.sh +helm template oan-my-service charts/oan-my-service +``` + +See [`charts/oan-common/README.md`](charts/oan-common/README.md) for the full +helper reference and +[`charts/oan-template/README.md`](charts/oan-template/README.md) for the +step-by-step adaptation guide. + +## Validation + +```bash +./scripts/lint-charts.sh +``` + +Runs `helm lint --strict` on every chart and `helm template` on every +application chart. CI runs the identical script +([`.github/workflows/helm-lint.yml`](.github/workflows/helm-lint.yml)) on pull +requests and on pushes to `main` and `development`. + +Because charts depend on `oan-common` through `file://../oan-common` and the +packaged dependency is not committed, an edit to `oan-common` only reaches a +consuming chart after `helm dependency update charts/` — or a run of the +lint script, which does it for you. + +## Conventions + +Chart naming, `version`/`appVersion` rules, changelog requirements, what every +service chart must declare, and how secrets are handled are documented in +[`CONVENTIONS.md`](CONVENTIONS.md). + +Two of those rules are enforced at render time rather than at review time: a +chart with empty `resources` fails to render, and so does an enabled probe with +no handler or with more than one. + +## Secrets + +No secret value belongs in this repository, and **no chart here renders a +Secret**. Charts reference Secrets by name; creating them is deliberately left +outside the charts, so that decision is made once rather than per chart. See +[`CONVENTIONS.md`](CONVENTIONS.md#secrets). From 5ff2aab002b477fbd7cb8b56f8a8eae44e6580cb Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:12:37 +0530 Subject: [PATCH 55/81] feat: point the discovery chart at the published image [#53] CI publishes ghcr.io/openagrinet/discovery-service now. It used to build and scan an image and push nothing, which is the whole reason repository was empty and the render was made to fail on it. tag stays empty rather than pinned. Empty falls through to Chart.AppVersion, so the chart ships pointing at the app version it was written against, and an environment wanting a different build says so in its own values file. Pinning it here would only be right once the chart and the app stop versioning together. Records that the package is private, which the old comment had no reason to. A cluster needs a docker-registry secret named in pullSecrets, and without it the deploy looks clean while the pod sits in ImagePullBackOff -- the same late-surfacing failure the empty-repository guard exists to prevent. --- charts/discovery/examples/discovery.dev.yaml | 8 ++++--- charts/discovery/values.yaml | 23 +++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/charts/discovery/examples/discovery.dev.yaml b/charts/discovery/examples/discovery.dev.yaml index 48a0f8e..6fa09cd 100644 --- a/charts/discovery/examples/discovery.dev.yaml +++ b/charts/discovery/examples/discovery.dev.yaml @@ -12,13 +12,15 @@ # No fullnameOverride needed: the chart is named "discovery", so a release # named "discovery" already produces Service/discovery. -# TODO: no image is published for discovery-service yet - CI builds and scans -# one but pushes nothing. Fill this in with whatever the first published tag is; -# the render fails until then, on purpose. +# The package is private, so a cluster needs a docker-registry secret named in +# pullSecrets as well as this. Pinned rather than left to Chart.AppVersion, +# because a dev environment is where you want to run a build the chart was not +# written against. image: registry: ghcr.io repository: openagrinet/discovery-service tag: "0.1.0" + pullSecrets: [] app: # The network this deployment serves. It fills an empty diff --git a/charts/discovery/values.yaml b/charts/discovery/values.yaml index 90c698f..90aa4ff 100644 --- a/charts/discovery/values.yaml +++ b/charts/discovery/values.yaml @@ -27,11 +27,22 @@ replicaCount: 1 # --------------------------------------------------------------------------- # Image # -# repository is EMPTY on purpose. discovery-service's CI builds an image and -# scans it but pushes it nowhere, so there is no tag to default to; the compose -# stack builds from the working tree. The render fails while this is empty -# rather than producing "ghcr.io/:0.1.0", which Helm and the API server both -# accept and which only surfaces later as an ImagePullBackOff. +# ghcr.io/openagrinet/discovery-service, which CI now publishes -- it used to +# build and scan an image and push nothing, which is why this was empty. +# +# tag stays empty, and that is not an oversight: empty falls through to +# Chart.AppVersion, so the chart ships pointing at the app version it was +# written against, and an environment that wants a different build says so in +# its own values file. Pin it here only when the chart and the app stop +# versioning together. +# +# The package is PRIVATE. A cluster pulling it needs a docker-registry secret +# and that secret named in pullSecrets below -- otherwise the deploy looks +# clean and the pod sits in ImagePullBackOff: +# +# kubectl create secret docker-registry ghcr \ +# --docker-server=ghcr.io --docker-username= \ +# --docker-password= # # The runtime is gcr.io/distroless/static-debian12:nonroot - no shell, no # package manager, uid 65532. That is what makes the security context defaults @@ -39,7 +50,7 @@ replicaCount: 1 # --------------------------------------------------------------------------- image: registry: ghcr.io - repository: "" + repository: openagrinet/discovery-service tag: "" digest: "" pullPolicy: IfNotPresent From a9fd2dafe346d598109ab994aaf6b636aa10b5b3 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:36:10 +0530 Subject: [PATCH 56/81] feat: validate publish against the spec, and trim the extended allowlist [OpenAgriNet/network-adapter#4] Three things, all in the schema validators. VALIDATE PUBLISH. The publish module declared a schemaValidator and never ran it, because validateSchema was missing from its steps: and a plugin absent from that list never executes. It is in the pipeline now. This works: the spec does define /catalog/publish, the validator indexes it under the action these payloads send, and both publish bodies pass -- 51 assertions green with it on. TRIM THE ALLOWLIST to raw.githubusercontent.com. beckn.org and example.com came from the upstream sample and nothing here ever fetched from either. An allowlist is only worth having if it lists what is actually used. REPOINT @context at the published pack. The payloads named schemas.openagrinet.global, which does not resolve -- so the URL identified a schema nobody could fetch. They now name the pack in OpenAgriNet/network-specs over the raw CDN, which returns it. Used the raw host rather than a /blob/ URL because /blob/ serves GitHub's HTML page, not the file. EXTENDED VALIDATION STAYS OFF, and the reason is worth recording because it is not the obvious one. With the two changes above the fetch works end to end -- the pack downloads, the schema is found by @type, validation genuinely runs. It fails at a seam instead: this validator strips @context and @type before validating, reasoning they are JSON-LD plumbing it handles itself, while the pack lists @type in allOf[].required. So every resource is rejected for a missing @type that the payload did send and the validator removed. Confirmed from the adapter's own error, which echoes the object it checked with both keys absent. Nothing in config or payload can bridge that. Either the pack drops @type from required -- probably right, since it describes attributes and the validator owns the JSON-LD keys -- or the validator stops stripping it, which is upstream. The note on the flag says so, so the next person does not repeat the experiment. --- .../config/adapters/exp.yaml.tmpl | 2 +- .../config/adapters/network.yaml.tmpl | 2 +- .../config/adapters/provider.yaml.tmpl | 18 ++++++++++++++++-- .../OAN-dev-flow.postman_collection.json | 8 ++++---- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/docker-deployment/config/adapters/exp.yaml.tmpl index 3b5e645..35d2ea9 100644 --- a/docker-deployment/config/adapters/exp.yaml.tmpl +++ b/docker-deployment/config/adapters/exp.yaml.tmpl @@ -107,7 +107,7 @@ modules: extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + extendedSchema_allowedDomains: "raw.githubusercontent.com" router: id: router diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/docker-deployment/config/adapters/network.yaml.tmpl index 89f3700..de2306b 100644 --- a/docker-deployment/config/adapters/network.yaml.tmpl +++ b/docker-deployment/config/adapters/network.yaml.tmpl @@ -110,7 +110,7 @@ modules: extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + extendedSchema_allowedDomains: "raw.githubusercontent.com" router: id: router diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/docker-deployment/config/adapters/provider.yaml.tmpl index e1366b2..0468959 100644 --- a/docker-deployment/config/adapters/provider.yaml.tmpl +++ b/docker-deployment/config/adapters/provider.yaml.tmpl @@ -127,7 +127,7 @@ modules: extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + extendedSchema_allowedDomains: "raw.githubusercontent.com" # Generic: fetches, compiles and caches whatever the registry's mapping # URLs point at. Knows nothing about any provider. @@ -253,11 +253,22 @@ modules: type: url location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" cacheTTL: "3600" + # OFF, and not for the usual reason. The fetch works: @context points + # at the published pack and the allowlist permits that host. What + # fails is a mismatch at the seam -- this validator strips @context + # and @type before validating, on the reasoning that they are JSON-LD + # plumbing it handles itself, while the OAN pack lists @type in + # allOf[].required. So every resource is rejected for a missing @type + # that the payload did send and the validator removed. + # + # Nothing here can bridge that. Either the pack drops @type from + # required, or the validator stops stripping it. Base validation + # below is unaffected and stays on. extendedSchema_enabled: "false" extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + extendedSchema_allowedDomains: "raw.githubusercontent.com" router: id: router @@ -265,6 +276,9 @@ modules: routingConfig: /app/config/routing-provider.yaml steps: + # Declaring the validator above is not enough: a plugin missing from this + # list never runs, which is why publish went unvalidated until now. + - validateSchema # the Beckn v2 spec, plus each resource's own @context - addRoute # publish -> the network layer - sign # as this provider, so the network layer can verify diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json index 048c4d9..3d904dc 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json @@ -1050,7 +1050,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -1167,7 +1167,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -1223,7 +1223,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n \n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n \n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -1373,7 +1373,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" From 3d87de3127345e85b72089af281b9a54a97a51ee Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:42:36 +0530 Subject: [PATCH 57/81] refactor: rename docker-deployment to quick-start, lift the collection out [OpenAgriNet/network-adapter#4] Three moves and one mapping change. docker-deployment -> quick-start docker-deployment/mocks -> quick-start/mock-server docker-deployment/postman-collection -> postman-collection (repo root) "docker-deployment" described the technology, not the purpose; quick-start says what it is for. The collection moves out because it is not part of the compose stack -- it is what you point AT a stack, and its environment file exists precisely so it can be aimed somewhere else. "mocks" reads like test doubles living in the source tree; mock-server says it is a service you run. Twenty-three path references updated across README, Makefile, .env.example, stack.sh, setup.py, bootstrap-ubuntu.sh, the compose file, mock-server's own README and both collection files. Recorded as renames, so file history follows. MAPPINGS NO LONGER STATE @context. The response half hardcoded the pack URL, so it had to know which one is current and could contradict what the request declared. It reads it off the incoming select now -- every resource carries @context and @type, so there is nothing to invent. Verified by running the real response mapping against a real mock upstream response and a real select: three resources out, @context matching the request for both the old schemas.openagrinet.global identifier and the GitHub pack URL. ONE THING TO KNOW BEFORE PULLING THIS ON A RUNNING HOST. Compose derives its project name from the directory holding the compose file, so every volume is renamed with it: docker-deployment_registry-data becomes quick-start_registry-data, and the same for discovery-data, npm-data, npm-letsencrypt and hyperdx-data. Docker will not move them. A plain `make up` after this pull starts on EMPTY volumes -- an empty registry, an empty discovery catalogue, and an NPM with no proxy hosts or certificates. The migration is in README under "Renaming this directory". --- .../OAN-dev-flow.postman_collection.json | 4 +- .../OAN-dev.postman_environment.json | 4 +- .../README.md | 0 .../.env.example | 8 +-- {docker-deployment => quick-start}/.gitignore | 0 .../CERTIFICATES.md | 0 {docker-deployment => quick-start}/Makefile | 2 +- {docker-deployment => quick-start}/README.md | 8 +-- .../bin/bootstrap-ubuntu.sh | 4 +- .../bin/setup.py | 0 .../bin/stack.sh | 2 +- .../config/adapters/exp.yaml.tmpl | 0 .../config/adapters/network.yaml.tmpl | 0 .../config/adapters/provider.yaml.tmpl | 0 .../config/adapters/routing-exp.yaml | 0 .../config/adapters/routing-network.yaml | 0 .../config/adapters/routing-provider.yaml | 0 .../config/discovery/instance.yaml.example | 0 .../agmarknet/mandi-price.select.yaml | 8 ++- .../weather-observation.select.yaml | 15 ++++-- .../config/registry/imports/realm-export.json | 0 .../config/registry/schemas/Participant.json | 0 .../registry/schemas/ProviderSchema.json | 0 .../registry/schemas/SchemaRegistry.json | 0 .../reverse-proxy/npm-advanced/exp.conf | 0 .../reverse-proxy/npm-custom/http_top.conf | 0 .../npm-custom/server_proxy.conf | 0 .../docker-compose.yml | 54 +++++++++++++++---- .../mock-server}/README.md | 4 +- .../mock-server}/mockagmarknet/Dockerfile | 0 .../mock-server}/mockagmarknet/go.mod | 0 .../mock-server}/mockagmarknet/main.go | 0 .../mock-server}/mockimd/Dockerfile | 0 .../mock-server}/mockimd/go.mod | 0 .../mock-server}/mockimd/main.go | 0 registry/.env | 35 ++++++++++++ 36 files changed, 113 insertions(+), 35 deletions(-) rename {docker-deployment/postman-collection => postman-collection}/OAN-dev-flow.postman_collection.json (99%) rename {docker-deployment/postman-collection => postman-collection}/OAN-dev.postman_environment.json (96%) rename {docker-deployment/postman-collection => postman-collection}/README.md (100%) rename {docker-deployment => quick-start}/.env.example (95%) rename {docker-deployment => quick-start}/.gitignore (100%) rename {docker-deployment => quick-start}/CERTIFICATES.md (100%) rename {docker-deployment => quick-start}/Makefile (97%) rename {docker-deployment => quick-start}/README.md (99%) rename {docker-deployment => quick-start}/bin/bootstrap-ubuntu.sh (98%) rename {docker-deployment => quick-start}/bin/setup.py (100%) rename {docker-deployment => quick-start}/bin/stack.sh (99%) rename {docker-deployment => quick-start}/config/adapters/exp.yaml.tmpl (100%) rename {docker-deployment => quick-start}/config/adapters/network.yaml.tmpl (100%) rename {docker-deployment => quick-start}/config/adapters/provider.yaml.tmpl (100%) rename {docker-deployment => quick-start}/config/adapters/routing-exp.yaml (100%) rename {docker-deployment => quick-start}/config/adapters/routing-network.yaml (100%) rename {docker-deployment => quick-start}/config/adapters/routing-provider.yaml (100%) rename {docker-deployment => quick-start}/config/discovery/instance.yaml.example (100%) rename {docker-deployment => quick-start}/config/mappings/agmarknet/mandi-price.select.yaml (95%) rename {docker-deployment => quick-start}/config/mappings/mausamgram/weather-observation.select.yaml (95%) rename {docker-deployment => quick-start}/config/registry/imports/realm-export.json (100%) rename {docker-deployment => quick-start}/config/registry/schemas/Participant.json (100%) rename {docker-deployment => quick-start}/config/registry/schemas/ProviderSchema.json (100%) rename {docker-deployment => quick-start}/config/registry/schemas/SchemaRegistry.json (100%) rename {docker-deployment => quick-start}/config/reverse-proxy/npm-advanced/exp.conf (100%) rename {docker-deployment => quick-start}/config/reverse-proxy/npm-custom/http_top.conf (100%) rename {docker-deployment => quick-start}/config/reverse-proxy/npm-custom/server_proxy.conf (100%) rename {docker-deployment => quick-start}/docker-compose.yml (91%) rename {docker-deployment/mocks => quick-start/mock-server}/README.md (94%) rename {docker-deployment/mocks => quick-start/mock-server}/mockagmarknet/Dockerfile (100%) rename {docker-deployment/mocks => quick-start/mock-server}/mockagmarknet/go.mod (100%) rename {docker-deployment/mocks => quick-start/mock-server}/mockagmarknet/main.go (100%) rename {docker-deployment/mocks => quick-start/mock-server}/mockimd/Dockerfile (100%) rename {docker-deployment/mocks => quick-start/mock-server}/mockimd/go.mod (100%) rename {docker-deployment/mocks => quick-start/mock-server}/mockimd/main.go (100%) create mode 100644 registry/.env diff --git a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json similarity index 99% rename from docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json rename to postman-collection/OAN-dev-flow.postman_collection.json index 3d904dc..d23d122 100644 --- a/docker-deployment/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -1510,7 +1510,7 @@ }, { "key": "weatherMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml" + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/mausamgram/weather-observation.select.yaml" }, { "key": "mandiProviderId", @@ -1540,7 +1540,7 @@ }, { "key": "mandiMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml" + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/agmarknet/mandi-price.select.yaml" }, { "key": "networkId", diff --git a/docker-deployment/postman-collection/OAN-dev.postman_environment.json b/postman-collection/OAN-dev.postman_environment.json similarity index 96% rename from docker-deployment/postman-collection/OAN-dev.postman_environment.json rename to postman-collection/OAN-dev.postman_environment.json index 903884e..3a8be1c 100644 --- a/docker-deployment/postman-collection/OAN-dev.postman_environment.json +++ b/postman-collection/OAN-dev.postman_environment.json @@ -148,7 +148,7 @@ }, { "key": "weatherMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/mausamgram/weather-observation.select.yaml", "enabled": true, "type": "default" }, @@ -192,7 +192,7 @@ }, { "key": "mandiMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml", + "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/agmarknet/mandi-price.select.yaml", "enabled": true, "type": "default" }, diff --git a/docker-deployment/postman-collection/README.md b/postman-collection/README.md similarity index 100% rename from docker-deployment/postman-collection/README.md rename to postman-collection/README.md diff --git a/docker-deployment/.env.example b/quick-start/.env.example similarity index 95% rename from docker-deployment/.env.example rename to quick-start/.env.example index d4390a5..bf45f53 100644 --- a/docker-deployment/.env.example +++ b/quick-start/.env.example @@ -36,8 +36,8 @@ ADAPTER_IMAGE=ghcr.io/nisargabd/oan-adapter:latest DISCOVERY_IMAGE=ghcr.io/nisargabd/discovery-service:${TAG:-latest} -# The two mock upstreams. Their sources are in mocks/, to be built and -# published once rather than built here -- see mocks/README.md for the build +# The two mock upstreams. Their sources are in mock-server/, to be built and +# published once rather than built here -- see mock-server/README.md for the build # commands and for what they deliberately get wrong. MOCKIMD_IMAGE=ghcr.io/nisargabd/oan-mockimd:latest MOCKAGMARKNET_IMAGE=ghcr.io/nisargabd/oan-mockagmarknet:latest @@ -177,5 +177,5 @@ MOCKAGMARKNET_DAYS=2 # # To change a mapping: edit config/mappings/, push, and the next cache expiry # picks it up -- about a minute for the adapter plus a few for GitHub's CDN. -MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml -MANDI_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml +MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/mausamgram/weather-observation.select.yaml +MANDI_MAPPING_URL=https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/agmarknet/mandi-price.select.yaml diff --git a/docker-deployment/.gitignore b/quick-start/.gitignore similarity index 100% rename from docker-deployment/.gitignore rename to quick-start/.gitignore diff --git a/docker-deployment/CERTIFICATES.md b/quick-start/CERTIFICATES.md similarity index 100% rename from docker-deployment/CERTIFICATES.md rename to quick-start/CERTIFICATES.md diff --git a/docker-deployment/Makefile b/quick-start/Makefile similarity index 97% rename from docker-deployment/Makefile rename to quick-start/Makefile index 32b7b8e..1aae647 100644 --- a/docker-deployment/Makefile +++ b/quick-start/Makefile @@ -9,7 +9,7 @@ # make help the rest # # Anchored to this file's own directory rather than $(PWD), so `make -C -# docker-deployment up` works from the repo root. +# quick-start up` works from the repo root. STACK := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))bin/stack.sh diff --git a/docker-deployment/README.md b/quick-start/README.md similarity index 99% rename from docker-deployment/README.md rename to quick-start/README.md index 6b56dca..3ac2419 100644 --- a/docker-deployment/README.md +++ b/quick-start/README.md @@ -21,7 +21,7 @@ Running here: - **three adapters** — experience, network and provider. Same image, three configs. - **two mock upstreams** — one standing in for Mausamgram's forecast API, one - for Agmarknet's Vistaar prices. Sources in `mocks/`; they are pulled as + for Agmarknet's Vistaar prices. Sources in `mock-server/`; they are pulled as published images like everything else. They exist so the stack answers a select end to end out of the box, with no external API and no ngrok tunnel. Loopback only, and the adapter reaches them by compose service name rather @@ -790,10 +790,10 @@ config/ agmarknet/ response transformation, in JSONata. These are the files the adapters fetch over the raw CDN -- the served copy and the reviewable copy are one file -mocks/ +mock-server/ mockimd/ the two mock upstreams. Sources only: they are mockagmarknet/ pulled as published images like everything else. - See mocks/README.md for the build commands and + See mock-server/README.md for the build commands and for what each deliberately gets wrong. postman-collection/ the whole flow as a Postman collection, with the deployment's own values prefilled and no registry @@ -980,7 +980,7 @@ The account lives in the `npm-data` volume, and there is no reset flow. Recreate the volume and you also lose every proxy host and certificate. Back it up: ```sh -docker run --rm -v docker-deployment_npm-data:/data -v "$PWD":/backup \ +docker run --rm -v quick-start_npm-data:/data -v "$PWD":/backup \ alpine tar czf /backup/npm-data.tgz -C /data . ``` diff --git a/docker-deployment/bin/bootstrap-ubuntu.sh b/quick-start/bin/bootstrap-ubuntu.sh similarity index 98% rename from docker-deployment/bin/bootstrap-ubuntu.sh rename to quick-start/bin/bootstrap-ubuntu.sh index e682a81..519c6bf 100755 --- a/docker-deployment/bin/bootstrap-ubuntu.sh +++ b/quick-start/bin/bootstrap-ubuntu.sh @@ -3,7 +3,7 @@ # Everything an Ubuntu VM needs before `make up` will run. Idempotent -- safe to # re-run, and safe to run on a box that already has some of this. # -# curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts//docker-deployment/bin/bootstrap-ubuntu.sh | bash +# curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts//quick-start/bin/bootstrap-ubuntu.sh | bash # # or, once the repo is cloned: # @@ -111,7 +111,7 @@ cat <<'NEXT' Then: git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git - cd helmcharts/docker-deployment + cd helmcharts/quick-start cp .env.example .env && nano .env # change every credential make up diff --git a/docker-deployment/bin/setup.py b/quick-start/bin/setup.py similarity index 100% rename from docker-deployment/bin/setup.py rename to quick-start/bin/setup.py diff --git a/docker-deployment/bin/stack.sh b/quick-start/bin/stack.sh similarity index 99% rename from docker-deployment/bin/stack.sh rename to quick-start/bin/stack.sh index 05f7343..3a5993c 100755 --- a/docker-deployment/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -26,7 +26,7 @@ set -euo pipefail # Every path in here is relative to the compose directory, and `docker compose` # needs to find docker-compose.yml, so anchor to it rather than to $PWD. That -# makes `make -C docker-deployment up` work from anywhere. +# makes `make -C quick-start up` work from anywhere. ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" diff --git a/docker-deployment/config/adapters/exp.yaml.tmpl b/quick-start/config/adapters/exp.yaml.tmpl similarity index 100% rename from docker-deployment/config/adapters/exp.yaml.tmpl rename to quick-start/config/adapters/exp.yaml.tmpl diff --git a/docker-deployment/config/adapters/network.yaml.tmpl b/quick-start/config/adapters/network.yaml.tmpl similarity index 100% rename from docker-deployment/config/adapters/network.yaml.tmpl rename to quick-start/config/adapters/network.yaml.tmpl diff --git a/docker-deployment/config/adapters/provider.yaml.tmpl b/quick-start/config/adapters/provider.yaml.tmpl similarity index 100% rename from docker-deployment/config/adapters/provider.yaml.tmpl rename to quick-start/config/adapters/provider.yaml.tmpl diff --git a/docker-deployment/config/adapters/routing-exp.yaml b/quick-start/config/adapters/routing-exp.yaml similarity index 100% rename from docker-deployment/config/adapters/routing-exp.yaml rename to quick-start/config/adapters/routing-exp.yaml diff --git a/docker-deployment/config/adapters/routing-network.yaml b/quick-start/config/adapters/routing-network.yaml similarity index 100% rename from docker-deployment/config/adapters/routing-network.yaml rename to quick-start/config/adapters/routing-network.yaml diff --git a/docker-deployment/config/adapters/routing-provider.yaml b/quick-start/config/adapters/routing-provider.yaml similarity index 100% rename from docker-deployment/config/adapters/routing-provider.yaml rename to quick-start/config/adapters/routing-provider.yaml diff --git a/docker-deployment/config/discovery/instance.yaml.example b/quick-start/config/discovery/instance.yaml.example similarity index 100% rename from docker-deployment/config/discovery/instance.yaml.example rename to quick-start/config/discovery/instance.yaml.example diff --git a/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml b/quick-start/config/mappings/agmarknet/mandi-price.select.yaml similarity index 95% rename from docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml rename to quick-start/config/mappings/agmarknet/mandi-price.select.yaml index 2f0e70b..5492275 100644 --- a/docker-deployment/config/mappings/agmarknet/mandi-price.select.yaml +++ b/quick-start/config/mappings/agmarknet/mandi-price.select.yaml @@ -93,6 +93,12 @@ response: | : []; $selected := beckn.message.contract.commitments[0]; + + /* The caller's own @context, echoed back rather than restated here. A + mapping that hardcodes it has to be reissued whenever the pack URL + moves, and it can disagree with what the request actually declared. + Backticks because @ is an operator in JSONata. */ + $ctx := beckn.message.contract.commitments[0].resources[0].resourceAttributes.`@context`; $ra := $selected.resources[0].resourceAttributes; /* Bound once because it is used twice -- for a resource's own id and for @@ -159,7 +165,7 @@ response: | market's observation for one day, so one. */ "quantity": 1, "resourceAttributes": { - "@context": "https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld", + "@context": $ctx, "@type": "openagrinet:MandiPrice", "informationMode": "Direct", "subjectCategories": $ra.subjectCategories, diff --git a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml b/quick-start/config/mappings/mausamgram/weather-observation.select.yaml similarity index 95% rename from docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml rename to quick-start/config/mappings/mausamgram/weather-observation.select.yaml index 1baf294..c1d230c 100644 --- a/docker-deployment/config/mappings/mausamgram/weather-observation.select.yaml +++ b/quick-start/config/mappings/mausamgram/weather-observation.select.yaml @@ -24,10 +24,9 @@ # here by name and version rather than by a path, because a path pins a branch # and a branch moves. # -# @context is the canonical schemas.openagrinet.global identifier. In JSON-LD -# that is a name, not a fetch target -- it does not have to resolve today, and -# substituting a raw git URL that does would put an implementation detail on the -# wire and break every consumer when the branch is renamed. +# @context is not stated here at all. The response echoes whatever the request +# declared, so this file never has to know which pack URL is current and cannot +# contradict the caller. # # Direct mode requires observationType, source, location, generatedAt and # parameters. informationMode is what selects those requirements: a catalog @@ -158,6 +157,12 @@ response: | $selected := beckn.message.contract.commitments[0]; + /* The caller's own @context, echoed back rather than restated here. A + mapping that hardcodes it has to be reissued whenever the pack URL + moves, and it can disagree with what the request actually declared. + Backticks because @ is an operator in JSONata. */ + $ctx := beckn.message.contract.commitments[0].resources[0].resourceAttributes.`@context`; + /* Bound once because it is used twice -- for a resource's own id and for the offer's reference to it. Two copies of the same expression is how a dangling reference gets reintroduced. */ @@ -225,7 +230,7 @@ response: | consumer who validates. */ "quantity": 1, "resourceAttributes": { - "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@context": $ctx, "@type": "openagrinet:WeatherObservation", "informationMode": "Direct", "observationType": "Forecast", diff --git a/docker-deployment/config/registry/imports/realm-export.json b/quick-start/config/registry/imports/realm-export.json similarity index 100% rename from docker-deployment/config/registry/imports/realm-export.json rename to quick-start/config/registry/imports/realm-export.json diff --git a/docker-deployment/config/registry/schemas/Participant.json b/quick-start/config/registry/schemas/Participant.json similarity index 100% rename from docker-deployment/config/registry/schemas/Participant.json rename to quick-start/config/registry/schemas/Participant.json diff --git a/docker-deployment/config/registry/schemas/ProviderSchema.json b/quick-start/config/registry/schemas/ProviderSchema.json similarity index 100% rename from docker-deployment/config/registry/schemas/ProviderSchema.json rename to quick-start/config/registry/schemas/ProviderSchema.json diff --git a/docker-deployment/config/registry/schemas/SchemaRegistry.json b/quick-start/config/registry/schemas/SchemaRegistry.json similarity index 100% rename from docker-deployment/config/registry/schemas/SchemaRegistry.json rename to quick-start/config/registry/schemas/SchemaRegistry.json diff --git a/docker-deployment/config/reverse-proxy/npm-advanced/exp.conf b/quick-start/config/reverse-proxy/npm-advanced/exp.conf similarity index 100% rename from docker-deployment/config/reverse-proxy/npm-advanced/exp.conf rename to quick-start/config/reverse-proxy/npm-advanced/exp.conf diff --git a/docker-deployment/config/reverse-proxy/npm-custom/http_top.conf b/quick-start/config/reverse-proxy/npm-custom/http_top.conf similarity index 100% rename from docker-deployment/config/reverse-proxy/npm-custom/http_top.conf rename to quick-start/config/reverse-proxy/npm-custom/http_top.conf diff --git a/docker-deployment/config/reverse-proxy/npm-custom/server_proxy.conf b/quick-start/config/reverse-proxy/npm-custom/server_proxy.conf similarity index 100% rename from docker-deployment/config/reverse-proxy/npm-custom/server_proxy.conf rename to quick-start/config/reverse-proxy/npm-custom/server_proxy.conf diff --git a/docker-deployment/docker-compose.yml b/quick-start/docker-compose.yml similarity index 91% rename from docker-deployment/docker-compose.yml rename to quick-start/docker-compose.yml index 1e8f520..f01caa8 100644 --- a/docker-deployment/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -90,14 +90,42 @@ x-adapter: &adapter x-adapter-env: &adapter-env CONFIG_FILE: /app/config/adapter.yaml - # Sent if the image's OpenTelemetry SDK picks these up, ignored if it does - # not -- which is the state this has not been verified in either direction, - # because the adapter image is pulled and its source is not in this repo. - # Nothing here depends on the answer: an unroutable or absent collector - # makes an OTLP exporter drop spans, not fail a request. What IS decided is - # the destination, so there is one place to change it. - OTEL_EXPORTER_OTLP_ENDPOINT: ${OTLP_ENDPOINT:-http://hyperdx:4318} - OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + # OpenTelemetry. These ARE read: the otelsetup plugin builds gRPC exporters + # and the SDK applies its OTEL_EXPORTER_OTLP_* environment config before any + # option the plugin passes. + # + # OTEL_EXPORTER_OTLP_INSECURE IS LOAD-BEARING, and the reason is worth + # knowing because the symptom names TLS and the cause does not. + # + # The SDK derives transport security from the endpoint's SCHEME: + # + # switch u.Scheme { case "http", "unix": WithInsecure(); default: WithSecure() } + # + # "default" includes an EMPTY scheme. OTLP_ENDPOINT is a bare host:port, + # because that is what the plugin's own otlpEndpoint config field takes -- + # so without the scheme prefix below, or this flag, the exporter dials TLS + # at hyperdx's plaintext 4317 and logs, once per export interval: + # + # failed to upload metrics: ... authentication handshake failed: + # tls: first record does not look like a TLS handshake + # + # The flag is set explicitly rather than left to the scheme, so a future + # edit to the URL cannot silently turn TLS back on. + # + # This does not weaken anything that was encrypted: the collector is on + # oan-internal, reachable from nowhere else, and serves plaintext OTLP. + # + # There is also a bug underneath, in the plugin rather than here: it passes + # insecure credentials via WithDialOption, and NewGRPCConfig appends its own + # default TLS credentials AFTER user options, so gRPC takes the TLS ones and + # the plugin's setting is silently discarded. The env var is what actually + # controls this until that is fixed upstream. + # + # OTEL_EXPORTER_OTLP_PROTOCOL is deliberately absent. The plugin constructs + # gRPC exporters in code, so a protocol preference here is ignored and only + # suggests the transport is configurable from this file. + OTEL_EXPORTER_OTLP_ENDPOINT: http://${OTLP_ENDPOINT:-hyperdx:4317} + OTEL_EXPORTER_OTLP_INSECURE: "true" services: # ========================================================================== @@ -290,7 +318,11 @@ services: # flip OTEL_EXPORTER=otlp in .env once the exporter is wired and the # traces land in HyperDX with no other change here. OTEL_EXPORTER: ${OTEL_EXPORTER:-none} - OTEL_EXPORTER_OTLP_ENDPOINT: ${OTLP_ENDPOINT:-http://hyperdx:4318} + # Same scheme requirement as the adapters -- see the note on the + # x-adapter anchor. Inert while OTEL_EXPORTER is none, but wrong + # in a way that would only surface when someone turns it on. + OTEL_EXPORTER_OTLP_ENDPOINT: http://${OTLP_ENDPOINT:-hyperdx:4317} + OTEL_EXPORTER_OTLP_INSECURE: "true" OTEL_SERVICE_NAME: oan-discovery # config/common.yaml is baked into the image and is the reviewed default. # To override a setting, copy config/discovery/instance.yaml.example to @@ -319,8 +351,8 @@ services: # mock upstreams -- stand-ins for the real provider APIs, so the stack can be # exercised without their credentials. # - # Pulled like everything else. The sources are in mocks/ to be built and - # published once, not built here -- see mocks/README.md. + # Pulled like everything else. The sources are in mock-server/ to be built and + # published once, not built here -- see mock-server/README.md. # # oan-internal only: an upstream is called BY the provider adapter and is # never reached from outside, so NPM has no business seeing it. The published diff --git a/docker-deployment/mocks/README.md b/quick-start/mock-server/README.md similarity index 94% rename from docker-deployment/mocks/README.md rename to quick-start/mock-server/README.md index e19165f..6b662ee 100644 --- a/docker-deployment/mocks/README.md +++ b/quick-start/mock-server/README.md @@ -14,8 +14,8 @@ The compose file **pulls** every image and builds nothing, so these are not built by `make up`. They are here to be built and published once, and then pulled like everything else: - docker build -t ghcr.io//oan-mockimd:latest mocks/mockimd - docker build -t ghcr.io//oan-mockagmarknet:latest mocks/mockagmarknet + docker build -t ghcr.io//oan-mockimd:latest mock-server/mockimd + docker build -t ghcr.io//oan-mockagmarknet:latest mock-server/mockagmarknet docker push ghcr.io//oan-mockimd:latest docker push ghcr.io//oan-mockagmarknet:latest diff --git a/docker-deployment/mocks/mockagmarknet/Dockerfile b/quick-start/mock-server/mockagmarknet/Dockerfile similarity index 100% rename from docker-deployment/mocks/mockagmarknet/Dockerfile rename to quick-start/mock-server/mockagmarknet/Dockerfile diff --git a/docker-deployment/mocks/mockagmarknet/go.mod b/quick-start/mock-server/mockagmarknet/go.mod similarity index 100% rename from docker-deployment/mocks/mockagmarknet/go.mod rename to quick-start/mock-server/mockagmarknet/go.mod diff --git a/docker-deployment/mocks/mockagmarknet/main.go b/quick-start/mock-server/mockagmarknet/main.go similarity index 100% rename from docker-deployment/mocks/mockagmarknet/main.go rename to quick-start/mock-server/mockagmarknet/main.go diff --git a/docker-deployment/mocks/mockimd/Dockerfile b/quick-start/mock-server/mockimd/Dockerfile similarity index 100% rename from docker-deployment/mocks/mockimd/Dockerfile rename to quick-start/mock-server/mockimd/Dockerfile diff --git a/docker-deployment/mocks/mockimd/go.mod b/quick-start/mock-server/mockimd/go.mod similarity index 100% rename from docker-deployment/mocks/mockimd/go.mod rename to quick-start/mock-server/mockimd/go.mod diff --git a/docker-deployment/mocks/mockimd/main.go b/quick-start/mock-server/mockimd/main.go similarity index 100% rename from docker-deployment/mocks/mockimd/main.go rename to quick-start/mock-server/mockimd/main.go diff --git a/registry/.env b/registry/.env new file mode 100644 index 0000000..3ae4e6b --- /dev/null +++ b/registry/.env @@ -0,0 +1,35 @@ +# Copy to .env and fill in. .env is gitignored; this file must stay secret-free. +# Vars marked REQUIRED have no default - compose refuses to start without them. + +RELEASE_VERSION=v2.0.0 +SCHEMA_DIR=schemas +DB_DIR=db-data + +# --- postgres --- +POSTGRES_DB=registry +POSTGRES_USER={POSTGRES_USER} +# REQUIRED. Baked into the data directory on first start: changing it later means +# deleting db-data/ and starting over. +POSTGRES_PASSWORD=devpassword123 + +# --- keycloak --- +KEYCLOAK_REALM=sunbird-rc +KEYCLOAK_ADMIN_USER=admin +# REQUIRED. Login for the Keycloak admin console. +KEYCLOAK_ADMIN_PASSWORD=admin +# REQUIRED. Keycloak admin console -> sunbird-rc realm -> Clients -> admin-api +# -> Credentials -> Regenerate Secret, then paste the value here. +KEYCLOAK_SECRET=689d57cb-20b9-4cfb-96c7-710bdd05e545 +KEYCLOAK_ADMIN_CLIENT_ID=admin-api +KEYCLOAK_CLIENT_ID=registry-frontend + +# --- registry --- +# REQUIRED. Default password the registry sets on Keycloak users it creates. +REGISTRY_DEFAULT_USER_PASSWORD=registrydev123 +# Role checks described in the API specification only apply while this is true. +AUTHENTICATION_ENABLED=true +# --- host ports (change if something already owns one) --- +REGISTRY_HOST_PORT=8081 +KEYCLOAK_HOST_PORT=8080 +KEYCLOAK_ADMIN_HOST_PORT=9990 +DB_HOST_PORT=5432 \ No newline at end of file From 6568f2eb6a879a8de4ae04c269f57f46d347f304 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:44:36 +0530 Subject: [PATCH 58/81] docs: document the volume rename this move causes, and fix stale paths [OpenAgriNet/network-adapter#4] The previous commit said the migration was in README under "Renaming this directory". It was not. It is now. Compose derives its project name from the directory holding the compose file, so renaming to quick-start renames all five volumes with it and Docker moves no data. A plain `make up` after the pull starts on empty ones: empty registry, empty catalogue, and an NPM with no proxy hosts or certificates. npm-letsencrypt is the one that hurts. Re-issuing runs into Let's Encrypt's five-per-week duplicate limit, so losing it can mean days without certificates. The section gives the copy-across, tested here on scratch volumes including a nested path before writing it down, and says to verify the registry and the proxy hosts before deleting the old copies. Also notes Keycloak's realm lives in registry-data, so it travels with that one volume and equally does not survive skipping it. Two stale references fixed while here: the collection is a sibling now rather than a subdirectory, and it was still described as six requests and 32 assertions -- it is nineteen and 50. --- quick-start/README.md | 69 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/quick-start/README.md b/quick-start/README.md index 3ac2419..cbe74fd 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -390,7 +390,7 @@ stops after step 3, which is enough to exercise the stack: ``` 1. registry and discovery (also registry-db, keycloak, discovery-db) 2. bin/setup.py keys, five registry participants, adapter configs -3. mocks, then the three adapters +3. mock upstreams, then the three adapters 4. nginx-proxy-manager the public edge -- 80 and 443, all interfaces 5. hyperdx ClickStack ``` @@ -547,10 +547,15 @@ Things worth knowing before editing any of this: ## Test it end to end -**Quickest path: import `postman-collection/`.** Six requests, 32 assertions, -nothing to fill in — publish, discover and select for both capabilities, with -every value already matching this deployment. A green run means the stack is -healthy rather than merely answering. +**Quickest path: import `../postman-collection/`.** Nineteen requests, 50 +assertions, nothing to fill in — the registry writes and reads that set the +stack up, then publish, discover and select for each capability, with every +value already matching this deployment. A green run means the stack is healthy +rather than merely answering. + +It sits at the repo root rather than in here, because it is not part of the +compose stack — it is what you point at one, and its environment file exists so +it can be aimed somewhere else. The rest of this section is one of those requests as curl, if you would rather see it than run it. @@ -795,9 +800,9 @@ mock-server/ mockagmarknet/ pulled as published images like everything else. See mock-server/README.md for the build commands and for what each deliberately gets wrong. -postman-collection/ the whole flow as a Postman collection, with the - deployment's own values prefilled and no registry - request in it +../postman-collection/ NOT in here -- a sibling of this directory. The + collection plus an environment file, because it is + what you point AT a stack rather than part of one keys/keys.json generated, gitignored. The private halves of the three adapter keypairs -- the one file here that is worth backing up, and the reason setup.py can @@ -984,6 +989,54 @@ docker run --rm -v quick-start_npm-data:/data -v "$PWD":/backup \ alpine tar czf /backup/npm-data.tgz -C /data . ``` +## Renaming this directory + +Worth knowing before you pull a rename onto a running host, because Docker will +not warn you. + +**Compose takes its project name from the directory holding the compose file**, +and every named volume is prefixed with it. So this directory becoming +`quick-start` renames all five: + + docker-deployment_registry-data -> quick-start_registry-data + docker-deployment_discovery-data -> quick-start_discovery-data + docker-deployment_npm-data -> quick-start_npm-data + docker-deployment_npm-letsencrypt -> quick-start_npm-letsencrypt + docker-deployment_hyperdx-data -> quick-start_hyperdx-data + +Docker does not move data between them. A plain `make up` after the pull starts +on **empty** volumes: an empty registry, an empty discovery catalogue, and an +NPM with no proxy hosts and no certificates. The old volumes are still there, +just orphaned. + +`npm-letsencrypt` is the one to care about. Re-issuing certificates means +Let's Encrypt's duplicate-certificate limit, five per week for the same set of +names, so losing it can leave you unable to get them back for days. + +**Copy the data across before starting.** Stop the stack first, from whichever +directory name it is currently running under: + +```sh +make down + +for v in registry-data discovery-data npm-data npm-letsencrypt hyperdx-data; do + docker volume create "quick-start_$v" >/dev/null + docker run --rm -v "docker-deployment_$v:/from" -v "quick-start_$v:/to" alpine \ + sh -c 'cd /from && tar cf - . | (cd /to && tar xf -)' +done +``` + +Then `make up`, and check the registry has its five participants and NPM still +lists your proxy hosts before deleting anything: + +```sh +docker volume ls | grep docker-deployment_ # the old copies, once you are sure +``` + +Keycloak shares `registry-data` with the registry, so its realm travels with +that one volume -- there is nothing separate to migrate, and equally nothing +that survives if you skip it. + ## Starting over ```sh From 25c01abc9f83732508b3e4841234de268aa1596b Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:46:56 +0530 Subject: [PATCH 59/81] fix: untrack registry/.env, which should never have been committed [OpenAgriNet/network-adapter#4] registry/ is local scratch -- a standalone registry someone ran by hand -- and its .env holds live credentials. It was never part of this repo. It got in because the rename commit used `git add -A` instead of naming paths, which swept up an untracked directory. The Permission denied warning on registry/db-data/ in that commit's output was the tell. Removed from tracking and added to a root .gitignore so the same command cannot do it again. REMOVING IT HERE DOES NOT UNDO IT. The file is still in 3d87de3, which is pushed, so POSTGRES_PASSWORD, KEYCLOAK_SECRET and REGISTRY_DEFAULT_USER_PASSWORD should be treated as disclosed and rotated. Two of the fields (KEYCLOAK_ADMIN_USER, KEYCLOAK_ADMIN_PASSWORD) match defaults already published in quick-start/.env.example, so those are no change. Purging it from history needs a rewrite and a force-push on a shared branch, which is not mine to do unilaterally -- rotation is the reliable fix regardless. --- .gitignore | 5 +++++ registry/.env | 35 ----------------------------------- 2 files changed, 5 insertions(+), 35 deletions(-) create mode 100644 .gitignore delete mode 100644 registry/.env diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..459f893 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ + +# Local scratch: a standalone registry someone ran by hand, with a real .env in +# it. Not part of this repo, and its .env holds live credentials -- it was +# committed once by a `git add -A` and must not be again. +registry/ diff --git a/registry/.env b/registry/.env deleted file mode 100644 index 3ae4e6b..0000000 --- a/registry/.env +++ /dev/null @@ -1,35 +0,0 @@ -# Copy to .env and fill in. .env is gitignored; this file must stay secret-free. -# Vars marked REQUIRED have no default - compose refuses to start without them. - -RELEASE_VERSION=v2.0.0 -SCHEMA_DIR=schemas -DB_DIR=db-data - -# --- postgres --- -POSTGRES_DB=registry -POSTGRES_USER={POSTGRES_USER} -# REQUIRED. Baked into the data directory on first start: changing it later means -# deleting db-data/ and starting over. -POSTGRES_PASSWORD=devpassword123 - -# --- keycloak --- -KEYCLOAK_REALM=sunbird-rc -KEYCLOAK_ADMIN_USER=admin -# REQUIRED. Login for the Keycloak admin console. -KEYCLOAK_ADMIN_PASSWORD=admin -# REQUIRED. Keycloak admin console -> sunbird-rc realm -> Clients -> admin-api -# -> Credentials -> Regenerate Secret, then paste the value here. -KEYCLOAK_SECRET=689d57cb-20b9-4cfb-96c7-710bdd05e545 -KEYCLOAK_ADMIN_CLIENT_ID=admin-api -KEYCLOAK_CLIENT_ID=registry-frontend - -# --- registry --- -# REQUIRED. Default password the registry sets on Keycloak users it creates. -REGISTRY_DEFAULT_USER_PASSWORD=registrydev123 -# Role checks described in the API specification only apply while this is true. -AUTHENTICATION_ENABLED=true -# --- host ports (change if something already owns one) --- -REGISTRY_HOST_PORT=8081 -KEYCLOAK_HOST_PORT=8080 -KEYCLOAK_ADMIN_HOST_PORT=9990 -DB_HOST_PORT=5432 \ No newline at end of file From 98e4e513b6ba400ad2d55e9671b802b4d50d50e7 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 11:50:23 +0530 Subject: [PATCH 60/81] refactor: drop the registry requests from the shared collection [OpenAgriNet/network-adapter#4] Back to what this collection is for: the flows a consumer drives. Two folders, Weather and Mandi, six requests, 32 assertions. The registry requests never belonged in a shared collection. The registry has no route through the gateway and publishes on loopback only, so they only ever worked from the VM or through a tunnel -- and bin/setup.py already seeds every row from the same .env the adapter configs are rendered from, which is what keeps the two from disagreeing. Anyone who does need to poke a row has the curl in quick-start's README. Removing them orphaned 29 variables -- the Keycloak and registry URLs and credentials, the node ids and keys, the binding keys, paths and mapping URLs. All dropped, from the collection and the environment file, so neither ships fields nothing reads. Six remain: the three adapter URLs, the two provider ids and networkId. networkAdapterUrl is kept although no request uses it. It is the other adapter a deployment exposes, and both its /publish and /discover verify signatures, so a network peer calls it directly -- Postman does not sign, so that call is not scripted, but the address is worth having in one place. The signing-key section moved to quick-start's README rather than being deleted with the requests it described. It is about what a registry row may contain -- that an upstream may hold keys, and why that lets a provider publish straight to the network adapter -- which outlives any particular Postman request. Reframed so it reads as a property of the row, not of a request that no longer exists. Verified against a live stack via an environment override: 6 requests, 32 assertions, no failures. --- .../OAN-dev-flow.postman_collection.json | 1143 +---------------- .../OAN-dev.postman_environment.json | 173 +-- postman-collection/README.md | 124 +- quick-start/README.md | 47 + 4 files changed, 74 insertions(+), 1413 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index d23d122..2aaf8d7 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -2,1025 +2,13 @@ "info": { "_postman_id": "7458e3b4-dd16-4ec1-84ce-006f2e423183", "name": "OAN dev \u2014 registry, publish, discover, select", - "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nTHREE FOLDERS: Registry, then one per capability. Registry issues the token the writes after it need; inside a capability folder Publish seeds the catalogue Discover looks for. So a first run goes top to bottom, and after that any folder -- or any one capability -- runs on its own.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", + "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nONE FOLDER PER CAPABILITY. Inside each, Publish seeds the catalogue Discover looks for, so run a folder top to bottom the first time; after that any request works on its own.\n\nTHERE ARE NO REGISTRY REQUESTS HERE, deliberately. The registry has no route through the gateway and publishes on loopback only, so nothing in a shared collection could reach it. bin/setup.py seeds all of it -- five participants and both capability bindings -- from the same .env the adapter configs are rendered from, which is what keeps the two from disagreeing.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "42114807" }, "item": [ { - "name": "1. Registry", - "item": [ - { - "name": "1. Get a write token", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const j = pm.response.json();", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "pm.test(\"a token was issued\", () => pm.expect(j.access_token).to.be.a(\"string\"));", - "pm.collectionVariables.set(\"token\", j.access_token);" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "X-Forwarded-Host", - "value": "keycloak:8080" - }, - { - "key": "X-Forwarded-Proto", - "value": "http" - } - ], - "body": { - "mode": "urlencoded", - "urlencoded": [ - { - "key": "client_id", - "value": "{{keycloakClientId}}" - }, - { - "key": "grant_type", - "value": "password" - }, - { - "key": "username", - "value": "{{registryUser}}" - }, - { - "key": "password", - "value": "{{registryPassword}}" - } - ] - }, - "url": { - "raw": "{{keycloakUrl}}/auth/realms/{{keycloakRealm}}/protocol/openid-connect/token", - "host": [ - "{{keycloakUrl}}" - ], - "path": [ - "auth", - "realms", - "{{keycloakRealm}}", - "protocol", - "openid-connect", - "token" - ] - }, - "description": "Every registry WRITE needs this. Searches take no token at all.\n\nTHE TWO X-Forwarded-* HEADERS ARE NOT OPTIONAL, and keycloak:8080 is the container-internal address on purpose -- not whatever port Keycloak is published on. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and every write below returns 401 with an empty body.\n\nSaved to {{token}}, so run this first.\n\nA 500 here usually means Keycloak's realm is missing -- it shares a database with the registry, so wiping the registry volume takes the realm with it. Restarting Keycloak re-imports it." - }, - "response": [] - }, - { - "name": "2. Create the exp node", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Same as the other creates, with one extra allowance: this collection ships", - "// the node key blank on deployments where it is not knowable, and an empty key", - "// fails schema validation before the duplicate check is ever reached. That is", - "// reported rather than failed.", - "const key = pm.variables.get(\"expNodeKey\") || \"\";", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, already present, or awaiting a key\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " if (msg.includes(\"duplicate key\")) return; // seeded already", - " if (!key && msg.includes(\"does not match pattern\")) {", - " console.log(\"not attempted: expNodeKey is empty -- paste it from keys/keys.json\");", - " return;", - " }", - " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{expNodeId}}\",\n \"name\": \"OAN experience layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{expNodeId}}\",\n \"role\": \"consumer\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{expNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant" - ] - }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, - "response": [] - }, - { - "name": "3. Create the network node", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Same as the other creates, with one extra allowance: this collection ships", - "// the node key blank on deployments where it is not knowable, and an empty key", - "// fails schema validation before the duplicate check is ever reached. That is", - "// reported rather than failed.", - "const key = pm.variables.get(\"networkNodeKey\") || \"\";", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, already present, or awaiting a key\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " if (msg.includes(\"duplicate key\")) return; // seeded already", - " if (!key && msg.includes(\"does not match pattern\")) {", - " console.log(\"not attempted: networkNodeKey is empty -- paste it from keys/keys.json\");", - " return;", - " }", - " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{networkNodeId}}\",\n \"name\": \"OAN network layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{networkNodeId}}\",\n \"role\": \"network\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{networkNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant" - ] - }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, - "response": [] - }, - { - "name": "4. Create the provider node", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Same as the other creates, with one extra allowance: this collection ships", - "// the node key blank on deployments where it is not knowable, and an empty key", - "// fails schema validation before the duplicate check is ever reached. That is", - "// reported rather than failed.", - "const key = pm.variables.get(\"providerNodeKey\") || \"\";", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, already present, or awaiting a key\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " if (msg.includes(\"duplicate key\")) return; // seeded already", - " if (!key && msg.includes(\"does not match pattern\")) {", - " console.log(\"not attempted: providerNodeKey is empty -- paste it from keys/keys.json\");", - " return;", - " }", - " pm.expect.fail(\"HTTP \" + pm.response.code + \": \" + msg);", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerNodeId}}\",\n \"name\": \"OAN provider layer adapter\",\n \"type\": \"node\",\n \"status\": \"active\",\n \"baseUrl\": \"https://{{providerNodeId}}\",\n \"role\": \"provider\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{providerNodeKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant" - ] - }, - "description": "A participant that speaks Beckn: an id, a role, and the public half of a signing keypair. This is the identity a signature is verified against.\n\nTHE KEY MUST MATCH keys/keys.json. bin/setup.py generates the keypair and seeds this row in one step, and that is the normal path -- this request exists to show what it writes. A node created here with a key the adapter does not hold produces signatures nobody can verify, and the id cannot be reclaimed afterwards.\n\nThe key is bare base64, no encoding label: the schema requires ^[A-Za-z0-9+/]{43}=$." - }, - "response": [] - }, - { - "name": "5. Create the weather upstream", - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The keys block below is OPTIONAL on an upstream, and blank by default.", - "//", - "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", - "// only conditional is `if type == \"node\" then require role and keys`, and it has", - "// no else -- so it adds requirements for a node and never forbids them on an", - "// upstream. The adapter accepts such a key as a signer too: the signature lookup", - "// filters on participantId alone and never compares type, and isSigning() treats", - "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", - "//", - "// WHY YOU WOULD WANT IT. With a key on this record the weather provider can sign its", - "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", - "// against this row. That takes the provider adapter out of the publish path", - "// entirely -- and with it the unauthenticated /publish it has to expose.", - "//", - "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", - "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", - "// refused for a reason that has nothing to do with what you were trying to do.", - "//", - "// Set weatherProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", - "// you create the record: the registry is append-only, so a partial PUT can add a", - "// keys array later but cannot remove or replace one.", - "const key = (pm.variables.get(\"weatherProviderKey\") || \"\").trim();", - "if (!key) {", - " const body = JSON.parse(pm.request.body.raw);", - " delete body.keys;", - " pm.request.body.update(JSON.stringify(body, null, 2));", - "}" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{providerId}}\",\n \"name\": \"IMD Mausamgram NWP (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{weatherBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{weatherProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant" - ] - }, - "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." - }, - "response": [] - }, - { - "name": "6. Create the weather capability binding", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{providerId}}|{{weatherCapability}}\",\n \"participantId\": \"{{providerId}}\",\n \"capabilityCode\": \"{{weatherCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/ProviderSchema", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "ProviderSchema" - ] - }, - "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." - }, - "response": [] - }, - { - "name": "7. Create the mandi upstream", - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The keys block below is OPTIONAL on an upstream, and blank by default.", - "//", - "// WHY IT IS ALLOWED. The Participant schema declares `keys` for every type; the", - "// only conditional is `if type == \"node\" then require role and keys`, and it has", - "// no else -- so it adds requirements for a node and never forbids them on an", - "// upstream. The adapter accepts such a key as a signer too: the signature lookup", - "// filters on participantId alone and never compares type, and isSigning() treats", - "// an absent `use` as signing (our schema drops `use`; `alg` carries the purpose).", - "//", - "// WHY YOU WOULD WANT IT. With a key on this record the mandi provider can sign its", - "// own catalogue and POST /publish directly to the NETWORK adapter, which verifies", - "// against this row. That takes the provider adapter out of the publish path", - "// entirely -- and with it the unauthenticated /publish it has to expose.", - "//", - "// WHY IT IS STRIPPED WHEN BLANK. The mocks have no keypair. An empty string fails", - "// the schema's ^[A-Za-z0-9+/]{43}=$ pattern, so sending the block empty would be", - "// refused for a reason that has nothing to do with what you were trying to do.", - "//", - "// Set mandiProviderKey to the PUBLIC half, bare base64, no encoding label. Do it when", - "// you create the record: the registry is append-only, so a partial PUT can add a", - "// keys array later but cannot remove or replace one.", - "const key = (pm.variables.get(\"mandiProviderKey\") || \"\").trim();", - "if (!key) {", - " const body = JSON.parse(pm.request.body.raw);", - " delete body.keys;", - " pm.request.body.update(JSON.stringify(body, null, 2));", - "}" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"participantId\": \"{{mandiProviderId}}\",\n \"name\": \"Agmarknet Vistaar (mock)\",\n \"type\": \"upstream\",\n \"status\": \"active\",\n \"baseUrl\": \"{{mandiBaseUrl}}\",\n \"keys\": [\n {\n \"alg\": \"ed25519\",\n \"key\": \"{{mandiProviderKey}}\",\n \"status\": \"active\",\n \"validFrom\": \"2026-01-01T00:00:00Z\",\n \"validUntil\": \"2030-01-01T00:00:00Z\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant" - ] - }, - "description": "An ordinary HTTP API the provider adapter calls. No role and no keys: it has never heard of Beckn, so it signs nothing and nothing verifies it.\n\nNo credential either -- the adapter presents one from its own config, which names the environment variable the value comes from. Nothing secret is ever held in the registry.\n\nbaseUrl is a compose service name because these are reached from inside the network." - }, - "response": [] - }, - { - "name": "8. Create the mandi capability binding", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// Append-only: no update on create, and a soft delete keeps the unique index,", - "// so a second run of this cannot succeed -- and must not fail the run either.", - "//", - "// The discriminator is the message, not the status code:", - "// 500 + \"duplicate key ...\" already there. Normal once setup.py has run.", - "// 400 + \"Validation Exception\" the body is wrong.", - "// 401 + empty body no usable token. See request 1.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) { /* 401 answers with no body */ }", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"created, or already present\", () => {", - " if (pm.response.code >= 200 && pm.response.code < 300) {", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - " return;", - " }", - " pm.expect(msg, \"HTTP \" + pm.response.code + \" and not a duplicate -- a real failure\")", - " .to.include(\"duplicate key\");", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"bindingKey\": \"{{mandiProviderId}}|{{mandiCapability}}\",\n \"participantId\": \"{{mandiProviderId}}\",\n \"capabilityCode\": \"{{mandiCapability}}\",\n \"status\": \"active\",\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{mandiPath}}\",\n \"mappings\": \"{{mandiMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/ProviderSchema", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "ProviderSchema" - ] - }, - "description": "Which upstream answers which capability, and how to call it: method, path, timeouts, and the mapping file URL.\n\nbindingKey is participantId|capabilityCode and it is the hinge of the whole thing. The provider adapter builds the same key from each incoming payload and a step answers only when its configured key matches. Change this without changing the adapter config and every select returns 404 NET_ENTITY_NOT_FOUND.\n\nactions is a list, not a map: the registry treats every nested object as an entity and injects an osid into it, which a map cannot carry." - }, - "response": [] - }, - { - "name": "9. Update the weather upstream URL (PUT)", - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The registry addresses a record by the osid it assigned on write, not by", - "// participantId. So an update has to look the osid up first. Doing it here rather", - "// than in a preceding request keeps this one runnable on its own.", - "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", - "const want = pm.variables.replaceIn(\"{{providerId}}\");", - "const filters = {}; filters[\"participantId\"] = { eq: want };", - "", - "pm.sendRequest({", - " url: base + \"/Participant/search\",", - " method: \"POST\",", - " header: { \"Content-Type\": \"application/json\" },", - " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", - "}, function (err, res) {", - " if (err) { console.log(\"osid lookup failed: \" + err); return; }", - " const rows = ((res.json() || {}).data) || [];", - " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", - " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", - "});" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// A partial body MERGES: fields you leave out keep their stored values. That is", - "// what makes this safe to re-run -- it writes the same value the variable already", - "// holds, so a default run changes nothing. Change the variable to change the row.", - "//", - "// Requires additionalProperties: true on the entity in config/registry/schemas/.", - "// The registry re-validates the MERGED document on update, and that document", - "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", - "// extraneous, with the field names it injected itself.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"update accepted\", () => {", - " if (msg.includes(\"extraneous key\")) {", - " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", - " + \"additionalProperties on this entity and restart the registry: \" + msg);", - " }", - " pm.expect(pm.response.code, msg).to.eql(200);", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - "});" - ] - } - } - ], - "request": { - "method": "PUT", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"baseUrl\": \"{{weatherBaseUrl}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant/{{targetOsid}}", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant", - "{{targetOsid}}" - ] - }, - "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." - }, - "response": [] - }, - { - "name": "10. Update the mandi upstream URL (PUT)", - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The registry addresses a record by the osid it assigned on write, not by", - "// participantId. So an update has to look the osid up first. Doing it here rather", - "// than in a preceding request keeps this one runnable on its own.", - "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", - "const want = pm.variables.replaceIn(\"{{mandiProviderId}}\");", - "const filters = {}; filters[\"participantId\"] = { eq: want };", - "", - "pm.sendRequest({", - " url: base + \"/Participant/search\",", - " method: \"POST\",", - " header: { \"Content-Type\": \"application/json\" },", - " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", - "}, function (err, res) {", - " if (err) { console.log(\"osid lookup failed: \" + err); return; }", - " const rows = ((res.json() || {}).data) || [];", - " if (!rows.length) { console.log(\"no Participant matching \" + want); return; }", - " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", - "});" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// A partial body MERGES: fields you leave out keep their stored values. That is", - "// what makes this safe to re-run -- it writes the same value the variable already", - "// holds, so a default run changes nothing. Change the variable to change the row.", - "//", - "// Requires additionalProperties: true on the entity in config/registry/schemas/.", - "// The registry re-validates the MERGED document on update, and that document", - "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", - "// extraneous, with the field names it injected itself.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"update accepted\", () => {", - " if (msg.includes(\"extraneous key\")) {", - " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", - " + \"additionalProperties on this entity and restart the registry: \" + msg);", - " }", - " pm.expect(pm.response.code, msg).to.eql(200);", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - "});" - ] - } - } - ], - "request": { - "method": "PUT", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"baseUrl\": \"{{mandiBaseUrl}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant/{{targetOsid}}", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant", - "{{targetOsid}}" - ] - }, - "description": "Repoint an upstream at a different address -- the request you want when a mock moves, or when a capability graduates to a real API on a new host.\n\nTHE BODY CARRIES ONLY baseUrl. A partial PUT merges: participantId, name, type and status keep their stored values. That is why this is safe to re-run -- it writes back the same value the variable already holds, so a default run changes nothing.\n\nThe osid in the URL is resolved by this request's pre-request script, because the registry addresses records by the id it assigned on write, not by participantId.\n\nNeeds additionalProperties: true on Participant in config/registry/schemas/ -- see the test script for why." - }, - "response": [] - }, - { - "name": "11. Update the weather binding's call plan (PUT)", - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "// The registry addresses a record by the osid it assigned on write, not by", - "// bindingKey. So an update has to look the osid up first. Doing it here rather", - "// than in a preceding request keeps this one runnable on its own.", - "const base = pm.variables.replaceIn(\"{{registryUrl}}\");", - "const want = pm.variables.replaceIn(\"{{weatherBindingKey}}\");", - "const filters = {}; filters[\"bindingKey\"] = { eq: want };", - "", - "pm.sendRequest({", - " url: base + \"/ProviderSchema/search\",", - " method: \"POST\",", - " header: { \"Content-Type\": \"application/json\" },", - " body: { mode: \"raw\", raw: JSON.stringify({ filters: filters }) }", - "}, function (err, res) {", - " if (err) { console.log(\"osid lookup failed: \" + err); return; }", - " const rows = ((res.json() || {}).data) || [];", - " if (!rows.length) { console.log(\"no ProviderSchema matching \" + want); return; }", - " pm.collectionVariables.set(\"targetOsid\", rows[0].osid);", - "});" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "// A partial body MERGES: fields you leave out keep their stored values. That is", - "// what makes this safe to re-run -- it writes the same value the variable already", - "// holds, so a default run changes nothing. Change the variable to change the row.", - "//", - "// Requires additionalProperties: true on the entity in config/registry/schemas/.", - "// The registry re-validates the MERGED document on update, and that document", - "// carries its own osid/osUpdatedAt/osOwner -- which a strict schema rejects as", - "// extraneous, with the field names it injected itself.", - "let body = {};", - "try { body = pm.response.json(); } catch (e) {}", - "const p = body.params || {};", - "const msg = p.errmsg || \"\";", - "", - "pm.test(\"update accepted\", () => {", - " if (msg.includes(\"extraneous key\")) {", - " pm.expect.fail(\"the schema still forbids extra properties -- relax \"", - " + \"additionalProperties on this entity and restart the registry: \" + msg);", - " }", - " pm.expect(pm.response.code, msg).to.eql(200);", - " pm.expect(p.status, msg).to.eql(\"SUCCESSFUL\");", - "});" - ] - } - } - ], - "request": { - "method": "PUT", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Authorization", - "value": "Bearer {{token}}" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"actions\": [\n {\n \"action\": \"select\",\n \"method\": \"GET\",\n \"path\": \"{{weatherPath}}\",\n \"mappings\": \"{{weatherMappingUrl}}\",\n \"timeoutMs\": 15000,\n \"retryMax\": 2,\n \"status\": \"active\"\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/ProviderSchema/{{targetOsid}}", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "ProviderSchema", - "{{targetOsid}}" - ] - }, - "description": "Change a call plan without recreating the binding: a new mapping URL, a different path, a longer timeout, more retries.\n\nUnlike the upstream update this DOES send the whole actions list, because actions is a list and replacing one entry means sending the list. So it also needs additionalProperties: true on ActionBinding, not just on ProviderSchema.\n\nRe-runnable for the same reason: it writes back what the variables already hold." - }, - "response": [] - }, - { - "name": "12. Search participants", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const rows = pm.response.json().data;", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "", - "// The three adapter identities must exist WITH keys, or nothing can sign.", - "const nodes = rows.filter(r => r.type === \"node\");", - "pm.test(\"three adapter identities, each with a signing key\", () => {", - " pm.expect(nodes.length).to.be.at.least(3);", - " nodes.forEach(n => pm.expect((n.keys || []).length, n.participantId).to.be.above(0));", - "});", - "", - "pm.test(\"both upstreams are registered as type upstream\", () => {", - " [pm.variables.get(\"providerId\"), pm.variables.get(\"mandiProviderId\")].forEach(id => {", - " const up = rows.filter(r => r.participantId === id);", - " pm.expect(up.length, \"no row for \" + id).to.eql(1);", - " pm.expect(up[0].type, id).to.eql(\"upstream\");", - " });", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"filters\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/Participant/search", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "Participant", - "search" - ] - }, - "description": "Search takes NO token -- it is the one registry call a network peer actually needs. That is also why the dev deployment keeps the whole service off the public edge: SunbirdRC uses POST for both reads and writes, so no method rule separates this from a create.\n\nAn empty filters object returns everything. Narrow it with e.g. {\"filters\":{\"participantId\":{\"eq\":\"exp.oan.dev\"}}}." - }, - "response": [] - }, - { - "name": "13. Search provider bindings", - "event": [ - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "const rows = pm.response.json().data;", - "pm.test(\"200\", () => pm.response.to.have.status(200));", - "const keys = rows.map(r => r.bindingKey);", - "", - "pm.test(\"both capability bindings exist\", () => {", - " pm.expect(keys).to.include(pm.variables.get(\"weatherBindingKey\"));", - " pm.expect(keys).to.include(pm.variables.get(\"mandiBindingKey\"));", - "});", - "", - "// The call plans differ, which is the point.", - "pm.test(\"each binding carries its own path\", () => {", - " const paths = {};", - " rows.forEach(r => { paths[r.bindingKey] = (r.actions || [])[0] && r.actions[0].path; });", - " pm.expect(paths[pm.variables.get(\"weatherBindingKey\")]).to.eql(pm.variables.get(\"weatherPath\"));", - " pm.expect(paths[pm.variables.get(\"mandiBindingKey\")]).to.eql(pm.variables.get(\"mandiPath\"));", - "});" - ] - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"filters\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{registryUrl}}/ProviderSchema/search", - "host": [ - "{{registryUrl}}" - ], - "path": [ - "ProviderSchema", - "search" - ] - }, - "description": "The call plans, one row per capability. The two differ in path and mapping URL and point at different upstreams -- which is what lets one provider adapter serve both without knowing anything about either." - }, - "response": [] - } - ], - "description": "Everything that talks to the registry: one write token, the records bin/setup.py already created, updates for the fields that change, and the two searches.\n\nRUN THE TOKEN FIRST. Requests 2-11 need it; it is saved to {{token}}. Searches need no token at all -- that is the one registry call a network peer makes, and the reason the whole service is kept off the public edge, since SunbirdRC uses POST for reads and writes alike.\n\nNone of this changes a seeded stack. The creates report \"already present\" because the registry is append-only, and the updates write back the value the variable already holds." - }, - { - "name": "2. Weather", + "name": "1. Weather", "item": [ { "name": "1. Publish", @@ -1191,7 +179,7 @@ "description": "The weather capability end to end: publish its catalogue, find it, then ask for a forecast.\n\nRUN IN ORDER the first time -- Publish seeds the catalogue Discover looks for. After that Select works on its own.\n\nSelect is the interesting one. It goes to the same endpoint on the same adapter as Mandi's, and a different domain package answers it: each provider step builds a binding key from the payload, serves the request if the key is its own, and passes it through untouched if not. A 404 NET_ENTITY_NOT_FOUND here means no step claimed this payload -- compare {{providerId}} and the capability @type against the bindings in Registry." }, { - "name": "3. Mandi", + "name": "2. Mandi", "item": [ { "name": "1. Publish", @@ -1398,42 +386,6 @@ } ], "variable": [ - { - "key": "registryUrl", - "value": "http://localhost:8081/api/v1", - "description": "Loopback. Tunnel to the VM if it is not this machine." - }, - { - "key": "keycloakUrl", - "value": "http://localhost:8080", - "description": "Issues the write token." - }, - { - "key": "keycloakRealm", - "value": "sunbird-rc" - }, - { - "key": "keycloakClientId", - "value": "registry-frontend" - }, - { - "key": "registryUser", - "value": "no-user" - }, - { - "key": "registryPassword", - "value": "no-user-password" - }, - { - "key": "token", - "value": "", - "description": "Set by request 1. Do not fill in by hand." - }, - { - "key": "targetOsid", - "value": "", - "description": "Set by the PUT requests' pre-request scripts. Do not fill in by hand." - }, { "key": "expAdapterUrl", "value": "http://localhost:9202", @@ -1447,108 +399,21 @@ { "key": "networkAdapterUrl", "value": "http://localhost:9201", - "description": "The network layer adapter -- the peer-facing surface. No request in this collection uses it: discover reaches it via the experience adapter, and publish via the provider adapter. It is here because it is the one adapter besides exp that a deployment exposes publicly, and because its /publish and /discover both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not scripted here." - }, - { - "key": "discoveryUrl", - "value": "http://localhost:8090" - }, - { - "key": "expNodeId", - "value": "exp.oan.dev" - }, - { - "key": "expNodeKey", - "value": "", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "networkNodeId", - "value": "network.oan.dev" - }, - { - "key": "networkNodeKey", - "value": "", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "providerNodeId", - "value": "provider.oan.dev" - }, - { - "key": "providerNodeKey", - "value": "", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." + "description": "Used by no request here, on purpose. Discover reaches this adapter through the experience adapter and publish through the provider adapter. It is listed because it is the other adapter a deployment exposes, and both its /publish and /discover verify signatures -- so a network peer calls it directly. Postman does not sign, so those calls are not scripted." }, { "key": "providerId", "value": "mausamgram-mock", "description": "The weather upstream. Half of its binding key." }, - { - "key": "weatherCapability", - "value": "openagrinet:WeatherObservation" - }, - { - "key": "weatherBindingKey", - "value": "mausamgram-mock|openagrinet:WeatherObservation", - "description": "Used to look up the binding's osid for the update." - }, - { - "key": "weatherProviderKey", - "value": "", - "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the weather provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." - }, - { - "key": "weatherBaseUrl", - "value": "http://mockimd:9100", - "description": "A compose service name: reached from inside the network." - }, - { - "key": "weatherPath", - "value": "/get-daily" - }, - { - "key": "weatherMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/mausamgram/weather-observation.select.yaml" - }, { "key": "mandiProviderId", "value": "agmarknet-mock", "description": "The mandi upstream. Half of its binding key." }, - { - "key": "mandiCapability", - "value": "openagrinet:MandiPrice" - }, - { - "key": "mandiBindingKey", - "value": "agmarknet-mock|openagrinet:MandiPrice" - }, - { - "key": "mandiProviderKey", - "value": "", - "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the mandi provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." - }, - { - "key": "mandiBaseUrl", - "value": "http://mockagmarknet:9101" - }, - { - "key": "mandiPath", - "value": "/v1/fetch-agmarknet-vistaar" - }, - { - "key": "mandiMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/agmarknet/mandi-price.select.yaml" - }, { "key": "networkId", "value": "oan-dev" - }, - { - "key": "domain", - "value": "oan-dev" } ] } diff --git a/postman-collection/OAN-dev.postman_environment.json b/postman-collection/OAN-dev.postman_environment.json index 3a8be1c..2cd8167 100644 --- a/postman-collection/OAN-dev.postman_environment.json +++ b/postman-collection/OAN-dev.postman_environment.json @@ -9,13 +9,6 @@ "type": "default", "description": "Takes unsigned requests -- the app is inside the trust boundary." }, - { - "key": "networkAdapterUrl", - "value": "http://localhost:9201", - "enabled": true, - "type": "default", - "description": "The network layer adapter -- the peer-facing surface. No request in this collection uses it: discover reaches it via the experience adapter, and publish via the provider adapter. It is here because it is the one adapter besides exp that a deployment exposes publicly, and because its /publish and /discover both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not scripted here." - }, { "key": "providerAdapterUrl", "value": "http://localhost:9200", @@ -23,89 +16,6 @@ "type": "default", "description": "Where publish enters." }, - { - "key": "registryUrl", - "value": "http://localhost:8081/api/v1", - "enabled": true, - "type": "default", - "description": "Loopback. Tunnel to the VM if it is not this machine." - }, - { - "key": "keycloakUrl", - "value": "http://localhost:8080", - "enabled": true, - "type": "default", - "description": "Issues the write token." - }, - { - "key": "discoveryUrl", - "value": "http://localhost:8090", - "enabled": true, - "type": "default" - }, - { - "key": "keycloakRealm", - "value": "sunbird-rc", - "enabled": true, - "type": "default" - }, - { - "key": "keycloakClientId", - "value": "registry-frontend", - "enabled": true, - "type": "default" - }, - { - "key": "registryUser", - "value": "no-user", - "enabled": true, - "type": "default" - }, - { - "key": "registryPassword", - "value": "no-user-password", - "enabled": true, - "type": "default" - }, - { - "key": "expNodeId", - "value": "exp.oan.dev", - "enabled": true, - "type": "default" - }, - { - "key": "expNodeKey", - "value": "", - "enabled": true, - "type": "default", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "networkNodeId", - "value": "network.oan.dev", - "enabled": true, - "type": "default" - }, - { - "key": "networkNodeKey", - "value": "", - "enabled": true, - "type": "default", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, - { - "key": "providerNodeId", - "value": "provider.oan.dev", - "enabled": true, - "type": "default" - }, - { - "key": "providerNodeKey", - "value": "", - "enabled": true, - "type": "default", - "description": "EMPTY ON PURPOSE. Paste the public half from keys/keys.json on the VM if you want to run this request. bin/setup.py already created this node, so it is normally not needed." - }, { "key": "providerId", "value": "mausamgram-mock", @@ -113,45 +23,6 @@ "type": "default", "description": "The weather upstream. Half of its binding key." }, - { - "key": "weatherCapability", - "value": "openagrinet:WeatherObservation", - "enabled": true, - "type": "default" - }, - { - "key": "weatherBindingKey", - "value": "mausamgram-mock|openagrinet:WeatherObservation", - "enabled": true, - "type": "default", - "description": "Used to look up the binding's osid for the update." - }, - { - "key": "weatherProviderKey", - "value": "", - "enabled": true, - "type": "default", - "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the weather provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." - }, - { - "key": "weatherBaseUrl", - "value": "http://mockimd:9100", - "enabled": true, - "type": "default", - "description": "A compose service name: reached from inside the network." - }, - { - "key": "weatherPath", - "value": "/get-daily", - "enabled": true, - "type": "default" - }, - { - "key": "weatherMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/mausamgram/weather-observation.select.yaml", - "enabled": true, - "type": "default" - }, { "key": "mandiProviderId", "value": "agmarknet-mock", @@ -159,43 +30,6 @@ "type": "default", "description": "The mandi upstream. Half of its binding key." }, - { - "key": "mandiCapability", - "value": "openagrinet:MandiPrice", - "enabled": true, - "type": "default" - }, - { - "key": "mandiBindingKey", - "value": "agmarknet-mock|openagrinet:MandiPrice", - "enabled": true, - "type": "default" - }, - { - "key": "mandiProviderKey", - "value": "", - "enabled": true, - "type": "default", - "description": "BLANK BY DEFAULT, and the request drops the keys block when it is. Set it to the mandi provider's PUBLIC signing key -- bare base64, ^[A-Za-z0-9+/]{43}=$ -- if that provider signs its own catalogues and publishes straight to the network adapter." - }, - { - "key": "mandiBaseUrl", - "value": "http://mockagmarknet:9101", - "enabled": true, - "type": "default" - }, - { - "key": "mandiPath", - "value": "/v1/fetch-agmarknet-vistaar", - "enabled": true, - "type": "default" - }, - { - "key": "mandiMappingUrl", - "value": "https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/config/mappings/agmarknet/mandi-price.select.yaml", - "enabled": true, - "type": "default" - }, { "key": "networkId", "value": "oan-dev", @@ -203,10 +37,11 @@ "type": "default" }, { - "key": "domain", - "value": "oan-dev", + "key": "networkAdapterUrl", + "value": "http://localhost:9201", "enabled": true, - "type": "default" + "type": "default", + "description": "Used by no request here, on purpose. Discover reaches this adapter through the experience adapter and publish through the provider adapter. It is listed because it is the other adapter a deployment exposes, and both its /publish and /discover verify signatures -- so a network peer calls it directly. Postman does not sign, so those calls are not scripted." } ], "_postman_variable_scope": "environment", diff --git a/postman-collection/README.md b/postman-collection/README.md index 367aba6..3bb1213 100644 --- a/postman-collection/README.md +++ b/postman-collection/README.md @@ -21,118 +21,32 @@ stay intact for the next person. No VM hostname or address is committed in either file. Deployment addresses are shared separately, and the environment file is the place to put them. -## Three folders +## Two folders, one per capability - 1. Registry 13 requests -- token, creates, updates, searches - 2. Weather 1. Publish 2. Discover 3. Select - 3. Mandi 1. Publish 2. Discover 3. Select + 1. Weather 1. Publish 2. Discover 3. Select + 2. Mandi 1. Publish 2. Discover 3. Select -Grouped by capability rather than by action, so one capability is one folder you -can run end to end: +Six requests, 32 assertions. Run a folder top to bottom the first time -- +Publish seeds the catalogue Discover looks for -- and after that any request +works on its own: - newman run OAN-dev-flow.postman_collection.json --folder "3. Mandi" + newman run OAN-dev-flow.postman_collection.json --folder "2. Mandi" -**Order is still position.** Registry issues the token the writes after it need, -and inside a capability folder Publish seeds the catalogue Discover looks for. -So a first run goes top to bottom; after that any folder runs on its own. +The two Select requests are the pair worth comparing. They hit the same +endpoint on the same adapter and different domain packages answer them, because +each provider step recognises its own binding key from the payload and passes +through anything else. Nothing routes by URL, path or domain. -The two Select requests are the pair worth comparing. They hit the same endpoint -on the same adapter and different domain packages answer them, because each -provider step recognises its own binding key from the payload and passes through -anything else. Nothing routes by URL, path or domain. +## No registry requests here -Each folder carries a description of what that leg does and what its failures -mean. +Deliberate. The registry has no route through the gateway and publishes on +loopback only, so nothing in a shared collection could reach it. +`bin/setup.py` seeds all of it -- five participants and both capability +bindings -- from the same `.env` the adapter configs are rendered from, which is +what keeps the two from disagreeing. -## A default run changes nothing - -That is deliberate, so the collection is safe to re-run against a live stack. - -- **The creates** report "already present". The registry is append-only -- no - update on create, and a soft delete keeps the unique index -- so a second run - cannot succeed. The test accepts a duplicate and fails anything else, so a - validation error or a bad token is still caught. -- **The updates** write back the same value the variable already holds. Change - a variable to actually change a row. - -## What the updates need - -`PUT /api/v1/{Entity}/{osid}` works, and a **partial body merges** -- send only -`baseUrl` and the name, type and status keep their stored values. - -Two things follow from how the registry implements it: - -- **The URL takes an osid, not a participantId.** The registry addresses a - record by the id it assigned on write. Each PUT resolves that itself in a - pre-request script, so the request still runs on its own. -- **The entity schema has to permit additional properties.** The registry - re-validates the *merged* document, and that document carries the `osid`, - `osUpdatedAt` and `osOwner` it injected itself -- which `additionalProperties: - false` rejects as extraneous, naming its own fields. `Participant`, - `ProviderSchema` and `ActionBinding` in `config/registry/schemas/` allow them - for this reason. `PublicKey` deliberately does not: nothing here updates key - material, and a partial PUT that omits `keys` never re-validates it. - -Schemas are read at startup, so a change there needs the registry restarted. - -## Filling in the node keys - -`expNodeKey`, `networkNodeKey` and `providerNodeKey` ship blank, because the -keypairs are generated per deployment into `keys/keys.json` on the host. -`bin/setup.py` already created those three rows, so requests 2-4 are normally -not needed at all -- they are here to show what a node record looks like. With -the keys blank they report "awaiting a key" rather than failing the run. - -If you do fill them in, use the public half exactly as `keys/keys.json` holds -it: bare base64, no encoding label. A node created with a key the adapter does -not hold produces signatures nobody can verify, and the id cannot be reclaimed. - -## Giving an upstream its own signing key - -The two upstream creates carry a `keys` block, blank by default, and drop it -when the variable is empty. Set `weatherProviderKey` or `mandiProviderKey` to a -provider's **public** signing key and the block is sent. - -**Why an upstream may have keys.** The `Participant` schema declares `keys` for -every type. Its only conditional is `if type == "node" then require role and -keys`, and it has no `else` -- so that branch adds requirements for a node and -never forbids keys on an upstream. The adapter accepts such a key as a signer -too: the signature lookup filters on `participantId` alone and never compares -`type`, and `isSigning()` treats an absent `use` as signing, which matters -because this schema drops `use` and lets `alg` carry the purpose. - -**Why you would want it.** With a key on that row the provider can sign its own -catalogue and `POST /publish` straight at the **network** adapter, which -verifies the signature against the row. The provider adapter drops out of the -publish path -- and with it the unauthenticated `/publish` it otherwise has to -expose, which is the whole reason the gateway carries a deny rule for that path. - -Verified end to end: an upstream record created with an `ed25519` key signed a -catalogue and the network adapter answered `catalog/on_publish` `ACCEPTED`, -while a wrong key, a body tampered with after signing, and a missing -`Authorization` header each came back `401`. - -The header a provider has to produce: - - Signature keyId="||ed25519", - algorithm="ed25519",created="",expires="", - headers="(created) (expires) digest",signature="" - -signed over exactly this string -- real newlines, and `BLAKE-512` meaning -BLAKE2b-512, not SHA: - - (created): - (expires): - digest: BLAKE-512= - -The `osid` is the one the registry assigns the key on write, so a provider has -to read it back from a `Participant/search` after registering. - -**Add the keys when you create the record.** A partial PUT can add a `keys` -array later but cannot remove or replace one -- the registry is append-only. And -the value is bare base64 matching `^[A-Za-z0-9+/]{43}=$`, no encoding label: a -`base64:` prefix left on the front fails verification later with a decode error -that points nowhere near the registry. +To look at a registry row, tunnel to the VM and use `Participant/search` +directly; the quick-start README has the curl. ## networkAdapterUrl diff --git a/quick-start/README.md b/quick-start/README.md index cbe74fd..09580a1 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -513,6 +513,53 @@ publishes it as. Keycloak builds the token's issuer from these headers and the registry validates that issuer against the internal address. Get it wrong and the registry rejects the token with a 401 and an empty body. +### Giving an upstream its own signing key + +An `upstream` row may carry a `keys` block. `bin/setup.py` does not add one -- +the mocks have no keypair -- but a real provider that signs its own catalogues +needs it, and the schema permits it. + +**Why an upstream may have keys.** The `Participant` schema declares `keys` for +every type. Its only conditional is `if type == "node" then require role and +keys`, and it has no `else` -- so that branch adds requirements for a node and +never forbids keys on an upstream. The adapter accepts such a key as a signer +too: the signature lookup filters on `participantId` alone and never compares +`type`, and `isSigning()` treats an absent `use` as signing, which matters +because this schema drops `use` and lets `alg` carry the purpose. + +**Why you would want it.** With a key on that row the provider can sign its own +catalogue and `POST /publish` straight at the **network** adapter, which +verifies the signature against the row. The provider adapter drops out of the +publish path -- and with it the unauthenticated `/publish` it otherwise has to +expose, which is the whole reason the gateway carries a deny rule for that path. + +Verified end to end: an upstream record created with an `ed25519` key signed a +catalogue and the network adapter answered `catalog/on_publish` `ACCEPTED`, +while a wrong key, a body tampered with after signing, and a missing +`Authorization` header each came back `401`. + +The header a provider has to produce: + + Signature keyId="||ed25519", + algorithm="ed25519",created="",expires="", + headers="(created) (expires) digest",signature="" + +signed over exactly this string -- real newlines, and `BLAKE-512` meaning +BLAKE2b-512, not SHA: + + (created): + (expires): + digest: BLAKE-512= + +The `osid` is the one the registry assigns the key on write, so a provider has +to read it back from a `Participant/search` after registering. + +**Add the keys when you create the record.** A partial PUT can add a `keys` +array later but cannot remove or replace one -- the registry is append-only. And +the value is bare base64 matching `^[A-Za-z0-9+/]{43}=$`, no encoding label: a +`base64:` prefix left on the front fails verification later with a decode error +that points nowhere near the registry. + ### Pointing a capability at a real API Two `.env` values and a re-run. To swap the weather mock for something real: From 18a92cacd13a736b0ce2714cd0615424751ada01 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 12:39:11 +0530 Subject: [PATCH 61/81] feat: give the catalogues a geometry and the discovers a spatial intent [OpenAgriNet/network-adapter#4] Discover asked only for text. It now sends all three parts of an Intent the Beckn spec defines -- textSearch, a jsonpath filter and an S_DWITHIN spatial constraint -- and the catalogues carry the geometry that makes the spatial part mean something. WHERE THE GEOMETRY GOES, because I had this wrong first. coverageAreas items are a oneOf -- an administrative area reference OR a GeoJSON geometry -- and coverageAreas is inherited from the AgricultureResource base pack, which neither WeatherObservation nor MandiPrice forbids in OnDemand mode. So a catalogue can carry the drawn area beside the code reference, exactly as the field's own description invites, with no provider-level field added. I had first concluded the geometry had to live at provider.availableAt[].geo, having read only the AdministrativeAreaReference branch of that oneOf and seen additionalProperties: false. coverageAreas is also the better target: it is per-resource, so two capabilities from one provider can cover different areas, where availableAt cannot distinguish them. resourceAttributes.location stays out of it -- the OnDemand branch explicitly forbids it, along with every other reading-bearing field. A catalogue advertises capability; a Direct select response carries the reading and its point. distanceMeters is 100000, not the 250000 the service's own docs example uses. The service caps it at 200000 and answers SCH_INVALID_FORMAT above that, which the test run found. VERIFIED THAT BOTH FILTERS ACTUALLY BITE, since one silently ignored looks identical to one that works: as written, 1 catalogue; probe point moved to London, 0; @type changed to a value we never publish, 0. Full run 51 assertions, no failures. Also reverts @context to the identifier the packs declare in their own x-jsonld block. Pointing it at the raw CDN was only ever to make extended validation fetch something, that stays off, and in JSON-LD @context is a name rather than a fetch target -- so the raw URL put a branch name on the wire for no gain. --- .../OAN-dev-flow.postman_collection.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 2aaf8d7..5388b84 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -38,7 +38,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -86,7 +86,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:WeatherObservation\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -155,7 +155,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -211,7 +211,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n \n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -272,7 +272,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\"\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:MandiPrice\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -361,7 +361,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" From 771408d4935168cd887b9c1b5bfbfd4781616456 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 12:47:50 +0530 Subject: [PATCH 62/81] feat: filter discover by networkId and schemaContext [OpenAgriNet/network-adapter#4] Both are Context fields, and both were absent, which made discover quieter than it looked. networkId absent does not mean "this network" -- the service reads its absence differently on each path: publish absent means APP_NETWORK_ID, used to fill an empty visibleTo discover absent means NO network predicate: every network's catalogs match So discover was searching across every network and getting the right answer by having only one. It now sends {{networkId}}, matching what publish stores. schemaContext goes in context, not message.intent. The reference implementation put it in intent, which Intent's additionalProperties:false forbids outright. Each entry is a context URI with an optional #fragment naming the type, and the predicate is exact string equality against what publish indexed off each resource: SchemaContext = resourceAttributes.@context SchemaType = resourceAttributes.@type So publish needs no schemaContext of its own -- which is also why reverting @context to the packs' declared identifier mattered: the filter has to spell the same string the resource carries, and a raw-CDN URL on one side would have drifted from the other. VERIFIED EACH PREDICATE ACTUALLY BITES, because a filter that matches everything is indistinguishable from one that works: as written 1 catalogue networkId -> another network 0 #fragment -> a type we never publish 0 context URI -> example.com 0 Full run 51 assertions, no failures. --- postman-collection/OAN-dev-flow.postman_collection.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 5388b84..12ef793 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -86,7 +86,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:WeatherObservation\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:WeatherObservation\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -272,7 +272,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\"\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:MandiPrice\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:MandiPrice\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" From 5bb139ba8362616bf10674759dee2e9646f0ebfe Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 12:54:30 +0530 Subject: [PATCH 63/81] feat: declare publishDirectives on the publish requests [OpenAgriNet/network-adapter#4] One directive per catalogue, in message.publishDirectives -- a sibling of catalogs, and permitted by the spec, which matters now that publish is schema-validated. catalogId matches the catalogue's own id; that is how a directive binds catalogType REGULAR. The spec requires this field, and MASTER means canonical definitions reusable across the network -- not this updateMode MERGE, stated rather than defaulted. FULL DELETES the resources a payload omits, so a republish would silently wipe anything unlisted. Worth spelling out so nobody flips it casually visibleTo ["{{networkId}}"], the same variable discover filters on visibleTo is the substantive one. Without it the network came from the receiving server's APP_NETWORK_ID, so the same publish landed in a different network on a differently-configured deployment, with no error and a discover that then found nothing. The payload is authoritative now, and both sides read one variable. Verified it binds rather than being decorative: published with visibleTo set to another network, and a discover scoped to local-network then returned 0 catalogues; republished with the correct value and it returned 1 again. Full run 51 assertions, no failures. resourceDirectives is deliberately absent -- Phase 1 refuses any carrying extends with SCH_TYPE_NOT_SUPPORTED, and there is nothing here to inherit. --- postman-collection/OAN-dev-flow.postman_collection.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 12ef793..4d96646 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -38,7 +38,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"cat-mausamgram-point-forecast-v2\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\",\n \"visibleTo\": [\n \"{{networkId}}\"\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -211,7 +211,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"cat-agmarknet-mandi-prices\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\",\n \"visibleTo\": [\n \"{{networkId}}\"\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" From 4d80200e6bff0d2f2d1c04f5c1029fe2ebd1302e Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 12:56:40 +0530 Subject: [PATCH 64/81] refactor: name the collection folders after the capability codes [OpenAgriNet/network-adapter#4] 1. Weather -> 1. WeatherObservation 2. Mandi -> 2. MandiPrice The folders now read as the capability codes they exercise -- the second half of every binding key, the @type on every resource, and the schema pack each one validates against. "Weather" and "Mandi" were shorthand that matched nothing else in the system. README updated, including the --folder example. Verified both folders still run standalone under the new names: WeatherObservation 13 assertions, MandiPrice 19, and the full run 51, none failing. --- postman-collection/OAN-dev-flow.postman_collection.json | 4 ++-- postman-collection/README.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 4d96646..f27f11c 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -8,7 +8,7 @@ }, "item": [ { - "name": "1. Weather", + "name": "1. WeatherObservation", "item": [ { "name": "1. Publish", @@ -179,7 +179,7 @@ "description": "The weather capability end to end: publish its catalogue, find it, then ask for a forecast.\n\nRUN IN ORDER the first time -- Publish seeds the catalogue Discover looks for. After that Select works on its own.\n\nSelect is the interesting one. It goes to the same endpoint on the same adapter as Mandi's, and a different domain package answers it: each provider step builds a binding key from the payload, serves the request if the key is its own, and passes it through untouched if not. A 404 NET_ENTITY_NOT_FOUND here means no step claimed this payload -- compare {{providerId}} and the capability @type against the bindings in Registry." }, { - "name": "2. Mandi", + "name": "2. MandiPrice", "item": [ { "name": "1. Publish", diff --git a/postman-collection/README.md b/postman-collection/README.md index 3bb1213..e1bf9c4 100644 --- a/postman-collection/README.md +++ b/postman-collection/README.md @@ -23,14 +23,14 @@ shared separately, and the environment file is the place to put them. ## Two folders, one per capability - 1. Weather 1. Publish 2. Discover 3. Select - 2. Mandi 1. Publish 2. Discover 3. Select + 1. WeatherObservation 1. Publish 2. Discover 3. Select + 2. MandiPrice 1. Publish 2. Discover 3. Select Six requests, 32 assertions. Run a folder top to bottom the first time -- Publish seeds the catalogue Discover looks for -- and after that any request works on its own: - newman run OAN-dev-flow.postman_collection.json --folder "2. Mandi" + newman run OAN-dev-flow.postman_collection.json --folder "2. MandiPrice" The two Select requests are the pair worth comparing. They hit the same endpoint on the same adapter and different domain packages answer them, because From 13a6a9726d2709e5dcb5d2a65392144dd8e56fd2 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 13:46:34 +0530 Subject: [PATCH 65/81] feat: validate resource attributes against their schema packs [OpenAgriNet/network-adapter#4] Base validation covers the Beckn envelope and treats resourceAttributes as a free-form object, so nothing checked a capability's own attributes. The provider adapter now runs extended validation on both its modules: it resolves each resource's @type to a capability schema and validates the object against it. This could not be switched on before. The validator stripped @context and @type as JSON-LD plumbing while the packs list @type in allOf[].required, so every resource was rejected for a missing @type the payload had sent -- the comment on the publish module said as much. The adapter now decides per key, keeping the ones a schema declares. The schemas are fetched, not committed. bin/fetch-schemas.sh downloads the eight published packs plus the five Beckn core schemas they $ref into config/schemas, which is gitignored and mounted read-only. A copy in git would drift from what the network publishes, which is the same reason the mappings are fetched. SCHEMA_PACKS_URL picks the revision, so that stays a deployment decision rather than something frozen into adapter config. The fetch runs in stack.sh step 2, with setup.py, not step 3. It has to: the provider adapter preloads these at startup and REFUSES TO START without them, and the path is a bind mount, so an adapter started first would find a directory Docker had invented and fail on it being empty. Failing closed is the behaviour to want here -- the alternative is accepting unvalidated payloads because a mount was forgotten. Resolution is a memory lookup, so no payload costs a network call and the container needs no egress to validate. The allowlist is narrowed to the host the packs' own @context names, so a local miss fails loudly rather than quietly fetching from elsewhere. Also renames the two capability steps to WeatherObservation and MandiPrice, matching the plugin ids on network-adapter feat/8-capability-schema-conformance -- the .so basename is the id this config refers to, so the two move together or the adapter fails with "unrecognized step". README: the schema validation section said extended was off and that publish was unvalidated. Both are now wrong. It also now records what extended validation does NOT check -- the library parses if/then/else and never evaluates it, so the packs' informationMode rules are unenforced and a pass here is not full pack conformance. --- quick-start/.env.example | 19 ++++ quick-start/.gitignore | 5 + quick-start/README.md | 60 ++++++++---- quick-start/bin/fetch-schemas.sh | 92 +++++++++++++++++++ quick-start/bin/stack.sh | 24 +++-- .../config/adapters/provider.yaml.tmpl | 63 ++++++++----- quick-start/docker-compose.yml | 6 ++ 7 files changed, 222 insertions(+), 47 deletions(-) create mode 100755 quick-start/bin/fetch-schemas.sh diff --git a/quick-start/.env.example b/quick-start/.env.example index bf45f53..e093821 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -93,6 +93,25 @@ OTEL_ENVIRONMENT=dev APP_NETWORK_ID=oan-dev BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml +# ---- schema packs ---------------------------------------------------------- +# The capability schemas the provider adapter's extended validation checks +# resourceAttributes against. bin/fetch-schemas.sh downloads them into +# config/schemas, mounted read-only at /app/config/schemas, and bin/stack.sh +# runs it before the adapters start -- a missing directory fails startup. +# +# Fetched rather than committed, like the mappings, and for the same reason: +# what this stack validates against has to be what the network publishes, not a +# copy here that drifts from it. +# +# Note the ref in the URL. Change it to move to a newer revision of the packs, +# or pin a tag so a deployment is not following a moving file. That choice +# belongs here and not in the adapter config, which names only the mount. +SCHEMA_PACKS_URL=https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema + +# The packs $ref five Beckn core schemas, fetched alongside them. Separate +# variable because they are the core's, on their own versions. +BECKN_SCHEMA_URL=https://schema.beckn.io + # ---- the three adapter identities ------------------------------------------ # bin/setup.py registers exactly these three in the registry and generates a # keypair for each. diff --git a/quick-start/.gitignore b/quick-start/.gitignore index d812134..bcf8876 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -10,6 +10,11 @@ config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml +# Downloaded by bin/fetch-schemas.sh from the published specs repository, and +# not committed for the same reason the mappings are not: a copy here would +# drift from what the network publishes. +config/schemas/ + # A local override for the discovery service, if you make one. config/discovery/instance.yaml diff --git a/quick-start/README.md b/quick-start/README.md index 09580a1..f0fdecf 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -716,7 +716,8 @@ the caller and re-signs. `select` never touches it: it goes straight to the provider adapter, which calls the upstream. Which upstream is not in any routing table. The provider adapter runs a chain -of capability steps — weather, then mandi — and each one builds a binding key +of capability steps — `WeatherObservation`, then `MandiPrice` — and each one +builds a binding key from the payload it is handed, serves the request if the key is its own, and passes it along untouched if not. The step that claims it looks the upstream up in the registry by that key. So one adapter fronts both capabilities, and a @@ -772,12 +773,31 @@ Three things about that: ## Schema validation Every adapter loads the pinned Beckn v2 LTS spec and validates request bodies -against it. The **extended** layer is off: it fetches each resource's own -`@context` and validates against that, which is a network call per payload and -a second thing that can fail. The `extendedSchema_*` settings in the configs -only take effect if it is switched on. - -Two consequences worth knowing before you write a payload: +against it. On the **provider adapter** a second layer runs too: it walks the +payload for objects carrying `@context` and `@type`, resolves the schema that +`@type` names, and validates the object against it. Base validation treats +`resourceAttributes` as a free-form object, so this is the only layer that +checks a capability's own attributes at all. + +The schemas are not in this repository. `bin/fetch-schemas.sh` downloads the +published packs into `config/schemas/`, mounted read-only at +`/app/config/schemas`, and `bin/stack.sh` runs it in step 2 — before the +adapters, because the provider adapter preloads them at startup and refuses to +start without them. `SCHEMA_PACKS_URL` in `.env` picks the revision; fetched +rather than committed for the same reason the mappings are, so what the stack +validates against is what the network publishes. + +Resolution is a memory lookup rather than a fetch per payload: everything under +that path is loaded once at startup and found by `@type`, so the container needs +no egress to validate and a `select` costs no extra round trip. + +**What it does not check.** The validator library parses `if`/`then`/`else` but +never evaluates it, so every pack rule predicated on `informationMode` is +unenforced — a pass here is not full conformance to a pack. It does enforce +types, string formats, `enum`, `const`, `required`, `additionalProperties`, +`not` and `allOf`/`anyOf`/`oneOf`. + +Three consequences worth knowing before you write a payload: - **Each resource under a commitment needs a `quantity`.** The spec's `Commitment.resources` requires `["id", "quantity"]` while `Resource` itself @@ -785,15 +805,16 @@ Two consequences worth knowing before you write a payload: a defect upstream, not something this deployment chose. Any value satisfies it. Without one, every `select` is refused with `SCH_REQUIRED_FIELD_MISSING: property "quantity" is missing`. -- **`publish` is not validated here, though it could be.** The two modules - that carry publishing — the provider adapter's root mount and the network - adapter — declare the validator but leave `validateSchema` out of their - `steps:`, and a plugin that is not in `steps:` never runs. That is a choice - in this config, not a limitation: the spec does define `/catalog/publish`, - the validator indexes it under the action `catalog/publish` that these - payloads send, and the collection's two publish bodies validate against it - with no errors. Turning it on is one line per module. It is off pending a - test rather than because it cannot work. +- **A `date-time` field will not take a bare date.** `validity.startsAt` and + `endsAt` are `format: date-time` in the packs, so `2025-08-20` is refused + and `2025-08-20T00:00:00+05:30` is accepted. `arrivalDate` is `format: date` + and wants the opposite. +- **`publish` is validated, on the provider adapter.** Declaring the validator + is not enough — a plugin missing from `steps:` never runs, which is why + publish went unchecked for a while — so `validateSchema` is in that module's + `steps:` and its resources are checked against their packs like any other. + The network adapter validates nothing: its single module runs + `validateSign`, `addRoute`, `sign` and never declares a validator. An action the spec does not know, or a body missing a required field, comes back as a signed NACK with a `SCH_*` code and the JSON path that failed. @@ -811,6 +832,8 @@ bin/ stack.sh the startup order, and why it is that order. Every make target is one line of delegation here. setup.py keys, five registry rows, the adapter configs + fetch-schemas.sh downloads the published schema packs that + extended validation resolves @type against config/ reverse-proxy/ npm-custom/ mounted to /data/nginx/custom, which NPM includes @@ -842,6 +865,9 @@ config/ agmarknet/ response transformation, in JSONata. These are the files the adapters fetch over the raw CDN -- the served copy and the reviewable copy are one file + schemas/ NOT in git. Downloaded by bin/fetch-schemas.sh + and mounted at /app/config/schemas, where the + provider adapter preloads them at startup mock-server/ mockimd/ the two mock upstreams. Sources only: they are mockagmarknet/ pulled as published images like everything else. @@ -907,7 +933,7 @@ capability at `...resources[].resourceAttributes.@type` — and comparing it against the key in its own config, which `setup.py` rendered from `.env`. Passing through is deliberate: it is what lets this one adapter serve both -weather and mandi. Compare the payload against `.env`, and re-run +capabilities. Compare the payload against `.env`, and re-run `bin/setup.py` plus `docker compose up -d --force-recreate provider-adapter` after changing `.env`. diff --git a/quick-start/bin/fetch-schemas.sh b/quick-start/bin/fetch-schemas.sh new file mode 100755 index 0000000..2e4b2b3 --- /dev/null +++ b/quick-start/bin/fetch-schemas.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Download the published schema packs into config/schemas. +# +# These are what the provider adapter's EXTENDED schema validation checks +# resourceAttributes against. Base validation covers the Beckn envelope and +# treats resourceAttributes as a free-form object; extended validation resolves +# each resource's @type to one of these documents and validates the object +# against it, which is what makes a wrong unit or a missing required attribute a +# rejected payload rather than something a provider discovers later. +# +# The layout below mirrors the published tree because that is what the adapter +# preloads: it walks localSchemaPath at STARTUP, keys every *.yaml by +# /attributes.yaml with the version segment dropped, and then looks up +# an object's @type -- the part after the colon -- directly. So no payload costs +# a network call, and the container needs no egress to validate. +# +# Not committed, and fetched rather than vendored, for the same reason the +# mappings are not vendored: a copy here would drift from what the network +# publishes, and what this stack validates against has to be what consumers +# actually read. +set -euo pipefail + +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +[ -f .env ] && { set -a; . ./.env; set +a; } + +: "${SCHEMA_PACKS_URL:?set SCHEMA_PACKS_URL in .env (see .env.example)}" +: "${BECKN_SCHEMA_URL:?set BECKN_SCHEMA_URL in .env (see .env.example)}" + +DEST=config/schemas + +# The OAN capability packs, each at v0.1. Listed rather than discovered: the set +# changes rarely, and an explicit list fails loudly when a name moves instead of +# silently fetching fewer schemas than the adapter needs. +OAN_PACKS=( + AgricultureResource # the base every capability below inherits from + AgricultureFacility + KnowledgeAdvisory + KnowledgeResource + MandiPrice + MarketIntelligence + WeatherAdvisory + WeatherObservation +) + +# The packs $ref these, so they have to be in the same directory. Without them +# the packs load but their references dangle, and a payload fails on a resolver +# error rather than on anything wrong with the payload. Versions are the ones +# the packs actually name -- Descriptor is v2.1 where the rest are v2.0. +BECKN_SCHEMAS=( + "Address/v2.0" + "Contact/v2.0" + "Descriptor/v2.1" + "GeoJSONGeometry/v2.0" + "Location/v2.0" +) + +fetch() { + local url="$1" out="$2" + mkdir -p "$(dirname "$out")" + # -f so an HTML 404 page is an error rather than a schema that fails to + # parse later; -L because schema.beckn.io redirects. + if ! curl -fsSL "$url" -o "$out"; then + echo "failed: $url" >&2 + return 1 + fi + printf ' %-56s %6s bytes\n' "$out" "$(wc -c <"$out")" +} + +echo "Fetching schema packs into $DEST/" +rm -rf "$DEST" + +for pack in "${OAN_PACKS[@]}"; do + fetch "${SCHEMA_PACKS_URL}/${pack}/v0.1/attributes.yaml" \ + "${DEST}/${pack}/v0.1/attributes.yaml" +done + +for ref in "${BECKN_SCHEMAS[@]}"; do + fetch "${BECKN_SCHEMA_URL}/${ref}/attributes.yaml" \ + "${DEST}/${ref}/attributes.yaml" +done + +count=$(find "$DEST" -name '*.yaml' | wc -l) +echo "Fetched $count schemas." + +# The adapter refuses to start on a missing directory, which is the behaviour to +# want -- the alternative is accepting unvalidated payloads because a mount was +# forgotten. An empty one only warns, so check here where the message is useful. +if [ "$count" -eq 0 ]; then + echo "no schemas fetched -- the provider adapter will reject every payload" >&2 + exit 1 +fi diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index 3a5993c..f13dc52 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -62,6 +62,9 @@ preflight() { python3 -c 'import cryptography' >/dev/null 2>&1 \ || die "the python 'cryptography' package is missing -- pip install cryptography" + + command -v curl >/dev/null 2>&1 \ + || die "curl is not installed -- bin/fetch-schemas.sh needs it" } # ------------------------------------------------------------------- up @@ -124,13 +127,20 @@ up_registry_tier() { docker compose up -d registry discovery } -# Generates the adapter keypairs, registers the three adapter identities, and +# Generates the adapter keypairs, registers the three adapter identities, # renders config/adapters/{provider,network,exp}.yaml from the .tmpl files -# beside them. Safe to re-run: keys come from keys/keys.json once it exists, -# and participants already registered are left alone. +# beside them, and downloads the schema packs. Safe to re-run: keys come from +# keys/keys.json once it exists, and participants already registered are left +# alone. +# +# The schemas have to be here rather than in step 3: the provider adapter +# preloads them at startup and REFUSES TO START without them, and the directory +# is bind-mounted, so an adapter started first would find a directory Docker +# invented and fail on an empty one. up_setup() { step 2 "$1" "bin/setup.py -- keys, five registry participants, adapter configs" python3 bin/setup.py + bash bin/fetch-schemas.sh } # Only now do the bind-mounted config files exist. @@ -309,12 +319,14 @@ restart_edge() { docker compose --profile reverse-proxy restart nginx-proxy-manager } -# Just step 2. Re-run it after editing a .tmpl, or to re-render configs that -# were deleted. It is idempotent, so this is always safe. +# Just step 2. Re-run it after editing a .tmpl, to re-render configs that were +# deleted, or to pick up a new SCHEMA_PACKS_URL. It is idempotent, so this is +# always safe. setup() { preflight step 1 1 "bin/setup.py" python3 bin/setup.py + bash bin/fetch-schemas.sh } usage() { @@ -326,7 +338,7 @@ bin/stack.sh up-core steps 1-3 only. Nothing public, no ClickHouse. down stop everything, keep the data destroy stop everything and DELETE every volume - setup re-run bin/setup.py only + setup re-run bin/setup.py and fetch the schema packs reverse-proxy start nginx-proxy-manager on its own (public, 80/443) observability start hyperdx on its own pull git pull, fixing the npm-custom ownership first diff --git a/quick-start/config/adapters/provider.yaml.tmpl b/quick-start/config/adapters/provider.yaml.tmpl index 0468959..6e00b89 100644 --- a/quick-start/config/adapters/provider.yaml.tmpl +++ b/quick-start/config/adapters/provider.yaml.tmpl @@ -112,22 +112,38 @@ modules: signer: id: signer - # Base Beckn v2 schema validation, against the pinned LTS spec. The - # extended layer is off: it fetches a resource's own @context and - # validates against that, which is a network call per payload and a - # second failure mode, and nothing here needs it yet. The allowed - # domains and cache settings below only take effect if it is turned on. + # Base Beckn v2 schema validation against the pinned LTS spec, plus the + # extended layer, which resolves each resource's @type to a capability + # schema and validates resourceAttributes against it. Base validation + # treats that object as free-form, so extended is the only layer that + # checks a capability's own attributes at all. + # + # Resolution is local: every schema under localSchemaPath is loaded at + # STARTUP and looked up by @type, so no payload costs a network call + # and this container needs no egress to validate. bin/fetch-schemas.sh + # populates it from the published specs repository and bin/stack.sh + # runs that before the adapters come up, because a missing directory + # fails startup rather than quietly accepting unvalidated payloads. + # + # Not enforced, and worth knowing before reading a pass as pack + # conformance: the validator library parses if/then/else but never + # evaluates it, so every pack rule predicated on informationMode is + # unchecked. schemaValidator: id: schemav2validator config: type: url location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" cacheTTL: "3600" - extendedSchema_enabled: "false" + extendedSchema_enabled: "true" + extendedSchema_localSchemaPath: "/app/config/schemas" + # The network fallback, reached only on a local miss. Restricted to + # the host the packs' own @context names, so a miss fails loudly + # instead of quietly fetching a schema from somewhere else. + extendedSchema_allowedDomains: "schemas.openagrinet.global" extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "raw.githubusercontent.com" # Generic: fetches, compiles and caches whatever the registry's mapping # URLs point at. Knows nothing about any provider. @@ -153,7 +169,7 @@ modules: # row per capability. Comma-separated, because a plugin config value is # a string; a binding key uses a pipe, so a comma is unambiguous. providerSteps: - - id: weather + - id: WeatherObservation config: bindingKeys: "__PROVIDER_BINDING_KEY__" authScheme: none @@ -165,7 +181,7 @@ modules: # Agmarknet takes its token as a QUERY parameter. The adapter holds # the parameter's name and the name of the variable carrying the # value, never the value -- and it redacts it from the URL it logs. - - id: mandi + - id: MandiPrice config: bindingKeys: "__MANDI_BINDING_KEY__" authScheme: query @@ -179,8 +195,8 @@ modules: steps: - validateSign # the sender's key, from the registry - validateSchema # the pinned Beckn v2 spec - - weather # openagrinet:WeatherObservation, or pass through - - mandi # openagrinet:MandiPrice, or pass through + - WeatherObservation # its binding key, or pass through + - MandiPrice # its binding key, or pass through - signAck # signs whatever the step answered with # The outbound leg: the provider's own catalogue system publishing to the @@ -253,22 +269,21 @@ modules: type: url location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" cacheTTL: "3600" - # OFF, and not for the usual reason. The fetch works: @context points - # at the published pack and the allowlist permits that host. What - # fails is a mismatch at the seam -- this validator strips @context - # and @type before validating, on the reasoning that they are JSON-LD - # plumbing it handles itself, while the OAN pack lists @type in - # allOf[].required. So every resource is rejected for a missing @type - # that the payload did send and the validator removed. - # - # Nothing here can bridge that. Either the pack drops @type from - # required, or the validator stops stripping it. Base validation - # below is unaffected and stays on. - extendedSchema_enabled: "false" + # ON. It could not be before: the validator stripped @context and + # @type as JSON-LD plumbing, while the packs list @type in + # allOf[].required -- so every resource was rejected for a missing + # @type the payload had sent. The validator now decides per key, + # keeping the ones a schema declares, which is what makes a + # catalogue's resource attributes checkable at all. + extendedSchema_enabled: "true" + extendedSchema_localSchemaPath: "/app/config/schemas" + # The network fallback, reached only on a local miss. Restricted to + # the host the packs' own @context names, so a miss fails loudly + # instead of quietly fetching a schema from somewhere else. + extendedSchema_allowedDomains: "schemas.openagrinet.global" extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" - extendedSchema_allowedDomains: "raw.githubusercontent.com" router: id: router diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index f01caa8..cdb4206 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -414,6 +414,12 @@ services: volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro + # The capability schemas extended validation resolves @type against. + # Preloaded at startup, so this container needs no egress to validate and + # no payload costs a fetch. bin/fetch-schemas.sh populates it from the + # published specs repository; a missing directory fails startup, which is + # why bin/stack.sh fetches before the adapters come up. + - ./config/schemas:/app/config/schemas:ro ports: - "127.0.0.1:${PROVIDER_ADAPTER_PORT}:9200" From 8aa8a2e86c40ff1c2222a321bcfe49024edce412 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 13:46:45 +0530 Subject: [PATCH 66/81] fix: give the mandi select a date-time validity [OpenAgriNet/network-adapter#4] validity.startsAt and endsAt resolve to ClosedTimePeriod -> TimePeriod, where both ends are format: date-time. The request sent bare dates, which base validation never looked at and extended validation refuses. Now full RFC 3339 timestamps in IST, the zone the market data is actually in, with the window closing at the end of the second day rather than at its start. Safe for the mapping: its request half reads the date with $substring($iso, 8, 2), which yields 20-08-2025 from a bare date and from a timestamp alike. Verified against the running stack -- the select passes extended validation and the collection is 51 of 51. Note arrivalDate wants the opposite: it is format: date, so the response's bare date there is correct and stays. --- postman-collection/OAN-dev-flow.postman_collection.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index f27f11c..0b8ed5d 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -361,7 +361,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20\",\n \"endsAt\": \"2025-08-21\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20T00:00:00+05:30\",\n \"endsAt\": \"2025-08-21T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" From 12817ef08260fc289490202bfd5cfc7035c4c2f4 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 14:17:35 +0530 Subject: [PATCH 67/81] refactor: resolve capability schemas from the payload's @context [OpenAgriNet/network-adapter#4] Drops bin/fetch-schemas.sh, the config/schemas mount, the two SCHEMA_* .env variables and the fetch step in stack.sh. All of it existed for one reason: the @context the payloads declared -- schemas.openagrinet.global -- does not resolve, and a failed schema fetch rejects the payload, so extended validation could only work off files placed on disk. The published packs do serve context.jsonld. Pointing @context at them makes the fetch the validator already knows how to do work, and the whole local mechanism unnecessary: @context .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld fetched .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml Better than the mount in two ways. A payload names the revision it wants to be judged against, so nothing in this repo can go stale against the published packs. And the deployment loses a step that could be forgotten -- there is no directory to populate and no startup that fails because it was not. The allowlist becomes raw.githubusercontent.com, the host @context actually resolves to, and is now load-bearing: an @context on any other host is refused before a fetch is attempted. The stack already reaches that host for the mappings, so no new egress. Costs, both in the README: the provider adapter needs that egress, and the first payload after a restart pays for the fetch -- about 2s, then cached for 24h. Verified in oan-local, same configs: publish and select pass, the first payload logs "fetching from network" and later ones "LRU cache hit", a foreign @context is refused with SCH_INVALID_JSONLD_CONTEXT, and the collection is 51 of 51. --- .../OAN-dev-flow.postman_collection.json | 12 +-- quick-start/.env.example | 19 ---- quick-start/.gitignore | 5 - quick-start/README.md | 35 +++---- quick-start/bin/fetch-schemas.sh | 92 ------------------- quick-start/bin/stack.sh | 24 ++--- .../config/adapters/provider.yaml.tmpl | 29 +++--- quick-start/docker-compose.yml | 6 -- 8 files changed, 44 insertions(+), 178 deletions(-) delete mode 100755 quick-start/bin/fetch-schemas.sh diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 0b8ed5d..9224e87 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -38,7 +38,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"cat-mausamgram-point-forecast-v2\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\",\n \"visibleTo\": [\n \"{{networkId}}\"\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"b1f0c2d3-4e5a-4b6c-8d9e-0f1a2b3c4d5e\",\n \"messageId\": \"c2e1d3f4-5a6b-4c7d-9e0f-1a2b3c4d5e6f\",\n \"timestamp\": \"2026-09-01T06:00:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-mausamgram-point-forecast-v2\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram point weather forecast\",\n \"shortDesc\": \"Five-day point weather forecast from IMD Mausamgram NWP\",\n \"longDesc\": \"Rainfall, temperature, humidity and wind forecast for a single point, five days ahead, from the India Meteorological Department's Mausamgram numerical weather prediction service.\"\n },\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"descriptor\": {\n \"code\": \"WX-POINT-FORECAST\",\n \"name\": \"Point weather forecast\",\n \"shortDesc\": \"Five-day weather forecast for a single point\",\n \"longDesc\": \"Daily rainfall, minimum and maximum temperature, minimum and maximum humidity and wind speed for a requested latitude and longitude.\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\",\n \"WindDirection\",\n \"Alert\"\n ],\n \"forecastHorizon\": \"P5D\",\n \"updateFrequency\": \"PT24H\",\n \"geographicGranularities\": [\n \"Point\"\n ],\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"cat-mausamgram-point-forecast-v2\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\",\n \"visibleTo\": [\n \"{{networkId}}\"\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -86,7 +86,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:WeatherObservation\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:WeatherObservation\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -155,7 +155,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:mausamgram:point-forecast\",\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:WeatherObservation\",\n \"subjectCategories\": [\n \"Weather\"\n ],\n \"location\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"informationMode\": \"OnDemand\",\n \"supportedObservationTypes\": [\n \"Forecast\"\n ],\n \"supportedParameters\": [\n \"Rainfall\",\n \"Temperature\",\n \"Humidity\",\n \"WindSpeed\"\n ],\n \"geographicGranularities\": [\n \"Point\"\n ]\n },\n \"quantity\": 1\n }\n ],\n \"offer\": {\n \"id\": \"offer:mausamgram:open-data\",\n \"resourceIds\": [\n \"res:mausamgram:point-forecast\"\n ],\n \"provider\": {\n \"id\": \"{{providerId}}\",\n \"descriptor\": {\n \"code\": \"IMD-NWP-01\",\n \"name\": \"IMD Mausamgram NWP\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -211,7 +211,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"cat-agmarknet-mandi-prices\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\",\n \"visibleTo\": [\n \"{{networkId}}\"\n ]\n }\n ]\n }\n}", + "raw": "{\n \"context\": {\n \"action\": \"catalog/publish\",\n \"version\": \"2.0.0\",\n \"transactionId\": \"d3a1b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"e4b2c3d5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:05:00Z\"\n },\n \"message\": {\n \"catalogs\": [\n {\n \"id\": \"cat-agmarknet-mandi-prices\",\n \"isActive\": true,\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet mandi prices\",\n \"shortDesc\": \"Daily commodity prices by market from Agmarknet\",\n \"longDesc\": \"Minimum, maximum and modal prices per commodity per market, reported daily by the Agmarknet Vistaar service.\"\n },\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n },\n \"validity\": {\n \"startDate\": \"2026-01-01T00:00:00Z\",\n \"endDate\": \"2027-12-31T23:59:59Z\"\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:mandi-price\",\n \"descriptor\": {\n \"code\": \"MANDI-PRICE\",\n \"name\": \"Mandi commodity price\",\n \"shortDesc\": \"Prices for a commodity at a named market\"\n },\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n },\n {\n \"code\": \"78\",\n \"name\": \"Tomato\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"historicalDataAvailable\": true,\n \"historyPeriod\": \"P1Y\",\n \"updateFrequency\": \"P1D\",\n \"languages\": [\n \"en\"\n ],\n \"coverageAreas\": [\n {\n \"codeScheme\": \"ISO-3166-1\",\n \"areaCode\": \"IN\",\n \"areaLevel\": \"Country\",\n \"areaName\": \"India\"\n },\n {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 68.0,\n 6.0\n ],\n [\n 98.0,\n 6.0\n ],\n [\n 98.0,\n 38.0\n ],\n [\n 68.0,\n 38.0\n ],\n [\n 68.0,\n 6.0\n ]\n ]\n ]\n }\n ]\n }\n }\n ]\n }\n ],\n \"publishDirectives\": [\n {\n \"catalogId\": \"cat-agmarknet-mandi-prices\",\n \"catalogType\": \"REGULAR\",\n \"updateMode\": \"MERGE\",\n \"visibleTo\": [\n \"{{networkId}}\"\n ]\n }\n ]\n }\n}", "options": { "raw": { "language": "json" @@ -272,7 +272,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:MandiPrice\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:MandiPrice\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -361,7 +361,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://schemas.openagrinet.global/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20T00:00:00+05:30\",\n \"endsAt\": \"2025-08-21T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"select\",\n \"networkId\": \"{{networkId}}\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\"\n },\n \"message\": {\n \"contract\": {\n \"commitments\": [\n {\n \"status\": {\n \"descriptor\": {\n \"code\": \"DRAFT\",\n \"name\": \"Draft\"\n }\n },\n \"resources\": [\n {\n \"id\": \"res:agmarknet:price-enquiry\",\n \"quantity\": 1,\n \"resourceAttributes\": {\n \"@context\": \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld\",\n \"@type\": \"openagrinet:MandiPrice\",\n \"informationMode\": \"OnDemand\",\n \"subjectCategories\": [\n \"Market\"\n ],\n \"supportedCommodities\": [\n {\n \"code\": \"2\",\n \"name\": \"Paddy(Common)\"\n }\n ],\n \"supportedPriceFields\": [\n \"Minimum\",\n \"Maximum\",\n \"Modal\"\n ],\n \"market\": {\n \"marketName\": \"Kasdol APMC\",\n \"marketCode\": \"2056\",\n \"district\": \"96\",\n \"state\": \"CG\"\n },\n \"validity\": {\n \"startsAt\": \"2025-08-20T00:00:00+05:30\",\n \"endsAt\": \"2025-08-21T23:59:59+05:30\"\n }\n }\n }\n ],\n \"offer\": {\n \"id\": \"offer:agmarknet:open-data\",\n \"resourceIds\": [\n \"res:agmarknet:price-enquiry\"\n ],\n \"provider\": {\n \"id\": \"{{mandiProviderId}}\",\n \"descriptor\": {\n \"code\": \"AGMARKNET-01\",\n \"name\": \"Agmarknet Vistaar\"\n }\n }\n }\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" diff --git a/quick-start/.env.example b/quick-start/.env.example index e093821..bf45f53 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -93,25 +93,6 @@ OTEL_ENVIRONMENT=dev APP_NETWORK_ID=oan-dev BECKN_SPEC_URL=https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml -# ---- schema packs ---------------------------------------------------------- -# The capability schemas the provider adapter's extended validation checks -# resourceAttributes against. bin/fetch-schemas.sh downloads them into -# config/schemas, mounted read-only at /app/config/schemas, and bin/stack.sh -# runs it before the adapters start -- a missing directory fails startup. -# -# Fetched rather than committed, like the mappings, and for the same reason: -# what this stack validates against has to be what the network publishes, not a -# copy here that drifts from it. -# -# Note the ref in the URL. Change it to move to a newer revision of the packs, -# or pin a tag so a deployment is not following a moving file. That choice -# belongs here and not in the adapter config, which names only the mount. -SCHEMA_PACKS_URL=https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema - -# The packs $ref five Beckn core schemas, fetched alongside them. Separate -# variable because they are the core's, on their own versions. -BECKN_SCHEMA_URL=https://schema.beckn.io - # ---- the three adapter identities ------------------------------------------ # bin/setup.py registers exactly these three in the registry and generates a # keypair for each. diff --git a/quick-start/.gitignore b/quick-start/.gitignore index bcf8876..d812134 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -10,11 +10,6 @@ config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml -# Downloaded by bin/fetch-schemas.sh from the published specs repository, and -# not committed for the same reason the mappings are not: a copy here would -# drift from what the network publishes. -config/schemas/ - # A local override for the discovery service, if you make one. config/discovery/instance.yaml diff --git a/quick-start/README.md b/quick-start/README.md index f0fdecf..60f70dc 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -624,7 +624,7 @@ curl -s -X POST http://127.0.0.1:9202/select \ "id": "res:mausamgram:point-forecast", "quantity": 1, "resourceAttributes": { - "@context": "https://schemas.openagrinet.global/schema/WeatherObservation/v0.1/context.jsonld", + "@context": "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld", "@type": "openagrinet:WeatherObservation", "subjectCategories": ["Weather"], "informationMode": "OnDemand", @@ -779,17 +779,25 @@ payload for objects carrying `@context` and `@type`, resolves the schema that `resourceAttributes` as a free-form object, so this is the only layer that checks a capability's own attributes at all. -The schemas are not in this repository. `bin/fetch-schemas.sh` downloads the -published packs into `config/schemas/`, mounted read-only at -`/app/config/schemas`, and `bin/stack.sh` runs it in step 2 — before the -adapters, because the provider adapter preloads them at startup and refuses to -start without them. `SCHEMA_PACKS_URL` in `.env` picks the revision; fetched -rather than committed for the same reason the mappings are, so what the stack -validates against is what the network publishes. +The schemas are not in this repository and are not mounted. Each resource's +`@context` names the published pack, and the validator swaps `context.jsonld` +for `attributes.yaml` to fetch the schema beside it: -Resolution is a memory lookup rather than a fetch per payload: everything under -that path is loaded once at startup and found by `@type`, so the container needs -no egress to validate and a `select` costs no extra round trip. +``` +@context .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld +fetched .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml +``` + +So a payload names the pack revision it wants to be judged against, and there is +no copy here to drift from the published one — the same reason the mappings are +fetched rather than committed. + +It is cached for 24 hours, so only the first payload after a restart pays for +the fetch. Two consequences: the provider adapter needs egress to +`raw.githubusercontent.com`, and a fetch that **fails rejects the payload** +rather than skipping validation. An `@context` on any other host is refused +before anything is fetched — `extendedSchema_allowedDomains` in the config is +the list. **What it does not check.** The validator library parses `if`/`then`/`else` but never evaluates it, so every pack rule predicated on `informationMode` is @@ -832,8 +840,6 @@ bin/ stack.sh the startup order, and why it is that order. Every make target is one line of delegation here. setup.py keys, five registry rows, the adapter configs - fetch-schemas.sh downloads the published schema packs that - extended validation resolves @type against config/ reverse-proxy/ npm-custom/ mounted to /data/nginx/custom, which NPM includes @@ -865,9 +871,6 @@ config/ agmarknet/ response transformation, in JSONata. These are the files the adapters fetch over the raw CDN -- the served copy and the reviewable copy are one file - schemas/ NOT in git. Downloaded by bin/fetch-schemas.sh - and mounted at /app/config/schemas, where the - provider adapter preloads them at startup mock-server/ mockimd/ the two mock upstreams. Sources only: they are mockagmarknet/ pulled as published images like everything else. diff --git a/quick-start/bin/fetch-schemas.sh b/quick-start/bin/fetch-schemas.sh deleted file mode 100755 index 2e4b2b3..0000000 --- a/quick-start/bin/fetch-schemas.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -# Download the published schema packs into config/schemas. -# -# These are what the provider adapter's EXTENDED schema validation checks -# resourceAttributes against. Base validation covers the Beckn envelope and -# treats resourceAttributes as a free-form object; extended validation resolves -# each resource's @type to one of these documents and validates the object -# against it, which is what makes a wrong unit or a missing required attribute a -# rejected payload rather than something a provider discovers later. -# -# The layout below mirrors the published tree because that is what the adapter -# preloads: it walks localSchemaPath at STARTUP, keys every *.yaml by -# /attributes.yaml with the version segment dropped, and then looks up -# an object's @type -- the part after the colon -- directly. So no payload costs -# a network call, and the container needs no egress to validate. -# -# Not committed, and fetched rather than vendored, for the same reason the -# mappings are not vendored: a copy here would drift from what the network -# publishes, and what this stack validates against has to be what consumers -# actually read. -set -euo pipefail - -cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -[ -f .env ] && { set -a; . ./.env; set +a; } - -: "${SCHEMA_PACKS_URL:?set SCHEMA_PACKS_URL in .env (see .env.example)}" -: "${BECKN_SCHEMA_URL:?set BECKN_SCHEMA_URL in .env (see .env.example)}" - -DEST=config/schemas - -# The OAN capability packs, each at v0.1. Listed rather than discovered: the set -# changes rarely, and an explicit list fails loudly when a name moves instead of -# silently fetching fewer schemas than the adapter needs. -OAN_PACKS=( - AgricultureResource # the base every capability below inherits from - AgricultureFacility - KnowledgeAdvisory - KnowledgeResource - MandiPrice - MarketIntelligence - WeatherAdvisory - WeatherObservation -) - -# The packs $ref these, so they have to be in the same directory. Without them -# the packs load but their references dangle, and a payload fails on a resolver -# error rather than on anything wrong with the payload. Versions are the ones -# the packs actually name -- Descriptor is v2.1 where the rest are v2.0. -BECKN_SCHEMAS=( - "Address/v2.0" - "Contact/v2.0" - "Descriptor/v2.1" - "GeoJSONGeometry/v2.0" - "Location/v2.0" -) - -fetch() { - local url="$1" out="$2" - mkdir -p "$(dirname "$out")" - # -f so an HTML 404 page is an error rather than a schema that fails to - # parse later; -L because schema.beckn.io redirects. - if ! curl -fsSL "$url" -o "$out"; then - echo "failed: $url" >&2 - return 1 - fi - printf ' %-56s %6s bytes\n' "$out" "$(wc -c <"$out")" -} - -echo "Fetching schema packs into $DEST/" -rm -rf "$DEST" - -for pack in "${OAN_PACKS[@]}"; do - fetch "${SCHEMA_PACKS_URL}/${pack}/v0.1/attributes.yaml" \ - "${DEST}/${pack}/v0.1/attributes.yaml" -done - -for ref in "${BECKN_SCHEMAS[@]}"; do - fetch "${BECKN_SCHEMA_URL}/${ref}/attributes.yaml" \ - "${DEST}/${ref}/attributes.yaml" -done - -count=$(find "$DEST" -name '*.yaml' | wc -l) -echo "Fetched $count schemas." - -# The adapter refuses to start on a missing directory, which is the behaviour to -# want -- the alternative is accepting unvalidated payloads because a mount was -# forgotten. An empty one only warns, so check here where the message is useful. -if [ "$count" -eq 0 ]; then - echo "no schemas fetched -- the provider adapter will reject every payload" >&2 - exit 1 -fi diff --git a/quick-start/bin/stack.sh b/quick-start/bin/stack.sh index f13dc52..3a5993c 100755 --- a/quick-start/bin/stack.sh +++ b/quick-start/bin/stack.sh @@ -62,9 +62,6 @@ preflight() { python3 -c 'import cryptography' >/dev/null 2>&1 \ || die "the python 'cryptography' package is missing -- pip install cryptography" - - command -v curl >/dev/null 2>&1 \ - || die "curl is not installed -- bin/fetch-schemas.sh needs it" } # ------------------------------------------------------------------- up @@ -127,20 +124,13 @@ up_registry_tier() { docker compose up -d registry discovery } -# Generates the adapter keypairs, registers the three adapter identities, +# Generates the adapter keypairs, registers the three adapter identities, and # renders config/adapters/{provider,network,exp}.yaml from the .tmpl files -# beside them, and downloads the schema packs. Safe to re-run: keys come from -# keys/keys.json once it exists, and participants already registered are left -# alone. -# -# The schemas have to be here rather than in step 3: the provider adapter -# preloads them at startup and REFUSES TO START without them, and the directory -# is bind-mounted, so an adapter started first would find a directory Docker -# invented and fail on an empty one. +# beside them. Safe to re-run: keys come from keys/keys.json once it exists, +# and participants already registered are left alone. up_setup() { step 2 "$1" "bin/setup.py -- keys, five registry participants, adapter configs" python3 bin/setup.py - bash bin/fetch-schemas.sh } # Only now do the bind-mounted config files exist. @@ -319,14 +309,12 @@ restart_edge() { docker compose --profile reverse-proxy restart nginx-proxy-manager } -# Just step 2. Re-run it after editing a .tmpl, to re-render configs that were -# deleted, or to pick up a new SCHEMA_PACKS_URL. It is idempotent, so this is -# always safe. +# Just step 2. Re-run it after editing a .tmpl, or to re-render configs that +# were deleted. It is idempotent, so this is always safe. setup() { preflight step 1 1 "bin/setup.py" python3 bin/setup.py - bash bin/fetch-schemas.sh } usage() { @@ -338,7 +326,7 @@ bin/stack.sh up-core steps 1-3 only. Nothing public, no ClickHouse. down stop everything, keep the data destroy stop everything and DELETE every volume - setup re-run bin/setup.py and fetch the schema packs + setup re-run bin/setup.py only reverse-proxy start nginx-proxy-manager on its own (public, 80/443) observability start hyperdx on its own pull git pull, fixing the npm-custom ownership first diff --git a/quick-start/config/adapters/provider.yaml.tmpl b/quick-start/config/adapters/provider.yaml.tmpl index 6e00b89..5fcf52e 100644 --- a/quick-start/config/adapters/provider.yaml.tmpl +++ b/quick-start/config/adapters/provider.yaml.tmpl @@ -118,12 +118,13 @@ modules: # treats that object as free-form, so extended is the only layer that # checks a capability's own attributes at all. # - # Resolution is local: every schema under localSchemaPath is loaded at - # STARTUP and looked up by @type, so no payload costs a network call - # and this container needs no egress to validate. bin/fetch-schemas.sh - # populates it from the published specs repository and bin/stack.sh - # runs that before the adapters come up, because a missing directory - # fails startup rather than quietly accepting unvalidated payloads. + # Resolution is a FETCH of the @context each resource declares. The + # validator swaps context.jsonld for attributes.yaml to get the schema + # beside it, so a payload names the pack revision it is judged + # against and no copy of the schemas is kept here to drift. Cached for + # the TTL below, so only the first payload after a restart pays; a + # fetch that FAILS rejects the payload rather than skipping + # validation, so this container does need egress to the allowed host. # # Not enforced, and worth knowing before reading a pass as pack # conformance: the validator library parses if/then/else but never @@ -136,11 +137,9 @@ modules: location: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" cacheTTL: "3600" extendedSchema_enabled: "true" - extendedSchema_localSchemaPath: "/app/config/schemas" - # The network fallback, reached only on a local miss. Restricted to - # the host the packs' own @context names, so a miss fails loudly - # instead of quietly fetching a schema from somewhere else. - extendedSchema_allowedDomains: "schemas.openagrinet.global" + # The host the packs' @context resolves to. An @context on any + # other host is refused before anything is fetched. + extendedSchema_allowedDomains: "raw.githubusercontent.com" extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" @@ -276,11 +275,9 @@ modules: # keeping the ones a schema declares, which is what makes a # catalogue's resource attributes checkable at all. extendedSchema_enabled: "true" - extendedSchema_localSchemaPath: "/app/config/schemas" - # The network fallback, reached only on a local miss. Restricted to - # the host the packs' own @context names, so a miss fails loudly - # instead of quietly fetching a schema from somewhere else. - extendedSchema_allowedDomains: "schemas.openagrinet.global" + # The host the packs' @context resolves to. An @context on any + # other host is refused before anything is fetched. + extendedSchema_allowedDomains: "raw.githubusercontent.com" extendedSchema_cacheTTL: "86400" extendedSchema_maxCacheSize: "100" extendedSchema_downloadTimeout: "30" diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index cdb4206..f01caa8 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -414,12 +414,6 @@ services: volumes: - ./config/adapters/provider.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-provider.yaml:/app/config/routing-provider.yaml:ro - # The capability schemas extended validation resolves @type against. - # Preloaded at startup, so this container needs no egress to validate and - # no payload costs a fetch. bin/fetch-schemas.sh populates it from the - # published specs repository; a missing directory fails startup, which is - # why bin/stack.sh fetches before the adapters come up. - - ./config/schemas:/app/config/schemas:ro ports: - "127.0.0.1:${PROVIDER_ADAPTER_PORT}:9200" From 493f8f54859568e6fc5327534b8c8ef3659ee6d4 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 14:45:56 +0530 Subject: [PATCH 68/81] docs: document updating a running deployment, and pin a matching adapter image [OpenAgriNet/network-adapter#4] Two gaps this closes, both found by upgrading a live VM. There was no upgrade path in the README. It covered a first run and starting over, with nothing in between, so a sync was reconstructed by hand each time. The new section splits it by what actually changed -- config only, config plus a new image, or payload shapes -- because those need different work and doing them in the wrong order is the likely way to break a working VM. It also gives the check order (steps initialized, then errors, then the collection) and the rollback, which is both halves or neither. ADAPTER_IMAGE pointed at a build that predates the plugin rename. The configs in this repo now name WeatherObservation and MandiPrice, and a plugin id is the basename of a .so inside the image, so the shipped default would have made every adapter exit with "unrecognized step: WeatherObservation" -- a failure that reads like a config typo and is not. It is pinned to a tag built from the adapter branch these configs expect, with the coupling written down next to it and a troubleshooting entry naming the error. The tag is in a personal namespace, as the others already are. An org package published by CI is the real fix; that is issue #5. --- quick-start/.env.example | 13 ++++++- quick-start/README.md | 76 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/quick-start/.env.example b/quick-start/.env.example index bf45f53..b53b584 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -33,7 +33,18 @@ # TAG pins the discovery service; unset means latest. Set it to deploy a known # build rather than whatever latest points at today: # TAG=v0.3.1 docker compose up -d -ADAPTER_IMAGE=ghcr.io/nisargabd/oan-adapter:latest +# +# THE ADAPTER IMAGE AND THE ADAPTER CONFIGS IN THIS REPO MOVE TOGETHER. The +# configs name plugins by id -- WeatherObservation, MandiPrice -- and an id is +# the basename of a .so inside the image. Point this at an image built before +# those were renamed and every adapter dies at startup with +# +# unrecognized step: WeatherObservation +# +# which reads like a config typo and is not. So pin a tag here rather than +# following latest: `docker compose ps` then says which build is running, and +# a rollback is this one line plus the matching `git checkout`. +ADAPTER_IMAGE=ghcr.io/ameersohel45/oan-adapter:0769807 DISCOVERY_IMAGE=ghcr.io/nisargabd/discovery-service:${TAG:-latest} # The two mock upstreams. Their sources are in mock-server/, to be built and diff --git a/quick-start/README.md b/quick-start/README.md index 60f70dc..ea978ba 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -443,6 +443,68 @@ That 403 is the check worth repeating after any NPM change: it is the only evidence that `npm-custom/server_proxy.conf` is still mounted, and losing the mount silently opens an unauthenticated catalogue write. +## Updating a deployment that is already running + +Three things can change, and they need different work. Getting this wrong is +the most likely way to break a working VM, so the order matters. + +**Config only** — a `.tmpl`, a routing file, `.env`. Re-render and recreate: + +```sh +make pull # git pull, and fixes the ownership NPM leaves behind +make up # step 2 re-renders the adapter configs, then recreates +``` + +`make restart` is not enough on its own for a `.tmpl` change: the adapters read +a rendered `.yaml`, and only `setup.py` writes it. + +**A new adapter image as well.** Any change to the plugin ids in +`config/adapters/*.tmpl` is this case, because an id is the basename of a `.so` +inside the image. Set `ADAPTER_IMAGE` to the matching tag BEFORE `make up`, or +the new config meets the old image and every adapter dies at startup. Build it +from the adapter repo at the commit the config expects: + +```sh +git clone https://github.com/OpenAgriNet/network-adapter.git +cd network-adapter && git checkout + +docker build -f Dockerfile.adapter-with-plugins \ + --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ + -t ghcr.io//oan-adapter:$(git rev-parse --short HEAD) . + +# the check worth doing before you push or deploy it +docker run --rm --entrypoint sh ghcr.io//oan-adapter: \ + -c 'ls plugins/ | grep -iE "weather|mandi"' +``` + +That last command should print the ids the config actually names. If it prints +something else, the image is from the wrong commit and nothing downstream will +work. + +**Payload shapes changed.** If `@context` moved, catalogues already in the +discovery database still carry the old value, and `discover` matches +`schemaContext` by exact string equality — so discover alone returns zero rows +against a database seeded before the change. Run the collection top to bottom +so publish reseeds first. `updateMode: MERGE` on the same `catalogId` updates +in place rather than duplicating. + +**Then check, in this order.** Cheapest first, because each failure explains +the next: + +```sh +docker compose logs provider-adapter | grep 'Processor steps initialized' +docker compose logs provider-adapter | grep -iE '"level":"(error|fatal)"' +make ps +``` + +The first should list the capability steps by the ids the config names. The +second should be empty. Only then run the collection. + +**Rolling back** is `git checkout `, `ADAPTER_IMAGE` back to the +old tag, `make up`. Both, together — the old image with the new config fails at +startup, and the new image with the old config starts but silently runs the old +behaviour. + ## What is in the registry, and why you did not create it `bin/setup.py` wrote all of it. Nothing in this section is a step to perform — @@ -940,6 +1002,20 @@ capabilities. Compare the payload against `.env`, and re-run `bin/setup.py` plus `docker compose up -d --force-recreate provider-adapter` after changing `.env`. +**Every adapter exits at startup with `unrecognized step: `.** Not a +config typo. A step name that is not one of the built-ins is looked up among +the loaded plugins, and a plugin's id is the basename of its `.so` in the +image — so this is `ADAPTER_IMAGE` pointing at a build that predates the name +in the config. Check what the image actually carries: + +```sh +docker run --rm --entrypoint sh $ADAPTER_IMAGE -c 'ls plugins/*.so' +``` + +Fix the tag, do not rename the step to match an old image — the config and the +image are meant to move together. See "Updating a deployment that is already +running". + **404 naming a binding with no active record.** The other side. A step *is* configured for the key, and it got as far as asking the registry which upstream answers it — but there is no active `ProviderSchema` row with that From acb19d2a6ffaa9295dcc7be6b42be1a127f13e4c Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 15:47:05 +0530 Subject: [PATCH 69/81] chore: follow latest for the adapter image [OpenAgriNet/network-adapter#4] ADAPTER_IMAGE was pinned to a SHA tag; it follows latest now. That needs two things said, because latest here behaves less dynamically than it reads. pull_policy is missing on every image and nothing in stack.sh pulls, so a tag already on disk is never re-fetched -- `make up` does NOT pick up a newer latest, and a stack can sit on a stale image indefinitely. Both the image comment and the upgrade section now give the explicit pull, and say why the pull without the recreate and the recreate without the pull each do nothing. The rollback note is corrected too. Rolling the config back is exact, but "the old image" has no name once the tag has moved, so recovery is by digest if the layers are even still on the box. Recorded rather than argued: pinning is still the thing to do before a change you might need to undo. The coupling warning is unchanged and matters more, not less -- an id is the basename of a .so inside the image, so latest moving under a deployment can produce "unrecognized step" with nothing in this repo having changed. --- quick-start/.env.example | 17 +++++++++++++---- quick-start/README.md | 28 ++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/quick-start/.env.example b/quick-start/.env.example index b53b584..7747a82 100644 --- a/quick-start/.env.example +++ b/quick-start/.env.example @@ -41,10 +41,19 @@ # # unrecognized step: WeatherObservation # -# which reads like a config typo and is not. So pin a tag here rather than -# following latest: `docker compose ps` then says which build is running, and -# a rollback is this one line plus the matching `git checkout`. -ADAPTER_IMAGE=ghcr.io/ameersohel45/oan-adapter:0769807 +# which reads like a config typo and is not. +# +# This follows latest, which behaves less dynamically than it reads: +# `pull_policy: missing` in docker-compose.yml means a tag already on disk is +# never re-fetched, so `make up` does NOT pick up a newer latest and a stack +# can sit on a stale image indefinitely. Taking a new one is deliberate: +# +# docker compose pull provider-adapter network-adapter exp-adapter +# docker compose up -d --force-recreate provider-adapter network-adapter exp-adapter +# +# Both halves matter: the pull without the recreate leaves the old container +# running, and the recreate without the pull recreates it on the old image. +ADAPTER_IMAGE=ghcr.io/ameersohel45/oan-adapter:latest DISCOVERY_IMAGE=ghcr.io/nisargabd/discovery-service:${TAG:-latest} # The two mock upstreams. Their sources are in mock-server/, to be built and diff --git a/quick-start/README.md b/quick-start/README.md index ea978ba..73a59ea 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -460,9 +460,20 @@ a rendered `.yaml`, and only `setup.py` writes it. **A new adapter image as well.** Any change to the plugin ids in `config/adapters/*.tmpl` is this case, because an id is the basename of a `.so` -inside the image. Set `ADAPTER_IMAGE` to the matching tag BEFORE `make up`, or -the new config meets the old image and every adapter dies at startup. Build it -from the adapter repo at the commit the config expects: +inside the image. The new config must not meet the old image, or every adapter +dies at startup. + +If `ADAPTER_IMAGE` names a **new tag**, set it before `make up` and that is +all. If it follows **`latest`**, `make up` alone is not enough: `pull_policy: +missing` means a tag already on disk is never re-fetched, and nothing in +`stack.sh` pulls, so the stack would quietly come back on the old image. Fetch +it explicitly first: + +```sh +docker compose pull provider-adapter network-adapter exp-adapter +``` + +Either way, build it from the adapter repo at the commit the config expects: ```sh git clone https://github.com/OpenAgriNet/network-adapter.git @@ -501,9 +512,14 @@ The first should list the capability steps by the ids the config names. The second should be empty. Only then run the collection. **Rolling back** is `git checkout `, `ADAPTER_IMAGE` back to the -old tag, `make up`. Both, together — the old image with the new config fails at -startup, and the new image with the old config starts but silently runs the old -behaviour. +old image, `make up`. Both, together — the old image with the new config fails +at startup, and the new image with the old config starts but silently runs the +old behaviour. + +Note this is the case `latest` serves badly. Rolling the config back is exact, +but "the old image" has no name if the tag has already moved, so you would be +recovering it by digest — `docker images --digests` on the VM, if it is still +there at all. Pin a tag before a change you might need to undo. ## What is in the registry, and why you did not create it From 6f2d79fd0239218e90ca998bd2449f85bb9ca0cc Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 16:00:49 +0530 Subject: [PATCH 70/81] refactor: filter discover on subjectCategories rather than @type [OpenAgriNet/network-adapter#4] The jsonpath filter compared resourceAttributes."@type" against the capability, which the request already narrows by: context.schemaContext carries the same @context and @type and is matched by exact equality before the filter runs. So the filter demonstrated the mechanism and narrowed nothing that was not already narrowed. subjectCategories is a different dimension -- Weather for one capability, Market for the other -- so the filter now does work the other predicates do not. Written as subjectCategories[*] == "..." because the field is an array. Lax mode would auto-unwrap it and the comparison is existential either way, but the explicit form says what is meant and behaves the same in strict mode. Equality keeps it indexable, which is what the service requires of a filter arriving without a text search or spatial constraint to narrow the corpus first -- ours carry both, but the expression should not depend on that. Verified against the running stack rather than by reading: "Weather" returns the one catalogue, a category nothing publishes returns none, and the other capability's category on this request returns none -- so the predicate discriminates on the value rather than passing everything. Collection is 51 of 51. --- postman-collection/OAN-dev-flow.postman_collection.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 9224e87..9ca753a 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -86,7 +86,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:WeatherObservation\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -272,7 +272,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.\\\"@type\\\" == \\\"openagrinet:MandiPrice\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" From 541dabd7c59a5e973b54157a9070624c569e3f49 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 16:14:59 +0530 Subject: [PATCH 71/81] style: write the spatial targets in the same notation as the filter [OpenAgriNet/network-adapter#4] The two path-shaped fields in a discover were spelled differently for no reason: filters.expression in dot notation and spatial.targets in bracket-quoted form. The bracket form was copied from the discovery service's own documentation, where it is the CANONICAL spelling -- the one the service stores -- not a required input. Both go through the same Canonicalise(), which reads .name and ['name'] alike and rewrites either into the stored form before the comparison. So the two spellings were never doing different things, and one request carrying both just invited the reader to look for a distinction that is not there. The distinction that IS real, and is worth keeping straight: filters.expression is EXECUTED, handed to PostgreSQL as @filter::jsonpath, which is why it can carry a predicate. spatial.targets is COMPARED, as the right-hand side of target_path = ANY($1) against the path the publish walker recorded, which is why it can only name a location. That also means a typo in targets fails as an empty result rather than an error -- verified, a pointer at serviceArea instead of coverageAreas returns zero rows quietly. Checked both spellings against the running stack before changing anything: bracket and dot targets return the identical one catalogue and one resource, a wrong pointer returns none, and after the change the spatial constraint still narrows -- the same request with a point in London returns none. Collection is 51 of 51. --- postman-collection/OAN-dev-flow.postman_collection.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/OAN-dev-flow.postman_collection.json index 9ca753a..91725bb 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/OAN-dev-flow.postman_collection.json @@ -86,7 +86,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44\",\n \"messageId\": \"7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22\",\n \"timestamp\": \"2026-08-30T06:12:01.330Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld#openagrinet:WeatherObservation\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"weather forecast\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Weather\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -272,7 +272,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$['catalogs'][*]['resources'][*]['resourceAttributes']['coverageAreas'][*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", + "raw": "{\n \"context\": {\n \"version\": \"2.0.0\",\n \"action\": \"discover\",\n \"transactionId\": \"a1b2c3d4-5e6f-4a7b-8c9d-0e1f2a3b4c5d\",\n \"messageId\": \"b2c3d4e5-6f7a-4b8c-9d0e-1f2a3b4c5d6e\",\n \"timestamp\": \"2026-09-01T06:10:00Z\",\n \"networkId\": \"{{networkId}}\",\n \"schemaContext\": [\n \"https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld#openagrinet:MandiPrice\"\n ]\n },\n \"message\": {\n \"intent\": {\n \"textSearch\": \"mandi commodity price\",\n \"filters\": {\n \"type\": \"jsonpath\",\n \"expression\": \"$.catalogs[*].resources[*] ? (@.resourceAttributes.subjectCategories[*] == \\\"Market\\\")\"\n },\n \"spatial\": [\n {\n \"op\": \"S_DWITHIN\",\n \"targets\": \"$.catalogs[*].resources[*].resourceAttributes.coverageAreas[*]\",\n \"geometry\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 73.7898,\n 19.9975\n ]\n },\n \"distanceMeters\": 100000,\n \"quantifier\": \"ANY\"\n }\n ]\n }\n }\n}", "options": { "raw": { "language": "json" From 6c9e5c09f931f9ae7c9aee3d9b01e724743cb172 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 16:45:18 +0530 Subject: [PATCH 72/81] refactor: rename the collection and environment files [OpenAgriNet/network-adapter#4] OAN-dev-flow.postman_collection.json -> api-collection.json OAN-dev.postman_environment.json -> local_postman_environment.json The old names carried both the tool and the environment in them, which dated badly on two counts: the collection is not dev-specific -- pointing it at another deployment is a URL change, which is what the environment file is for -- and Postman's own suffix convention says nothing a reader needs. Kept the .json on the environment file. Postman filters the import dialog by extension, so a file without it cannot be selected. Two things that named the old state and no longer did: - The collection's own info.name still read "registry, publish, discover, select". The registry requests were dropped several commits ago, so it advertised a folder that is not there. - quick-start/README.md promised nineteen requests and fifty assertions "the registry writes and reads that set the stack up, then publish, discover and select". It is six requests and thirty-two assertions, with no registry requests at all -- setup.py seeds it, because the registry has no route through the edge. Corrected, and it now says why they are absent rather than leaving their absence to look like an omission. Content is otherwise untouched: diffed the renamed file against the old one and only info.name differs. oan-local's collection, which targets the stack running here, is 51 of 51. --- postman-collection/README.md | 6 +++--- ...-flow.postman_collection.json => api-collection.json} | 2 +- ...n_environment.json => local_postman_environment.json} | 0 quick-start/README.md | 9 +++++---- 4 files changed, 9 insertions(+), 8 deletions(-) rename postman-collection/{OAN-dev-flow.postman_collection.json => api-collection.json} (99%) rename postman-collection/{OAN-dev.postman_environment.json => local_postman_environment.json} (100%) diff --git a/postman-collection/README.md b/postman-collection/README.md index e1bf9c4..740bb16 100644 --- a/postman-collection/README.md +++ b/postman-collection/README.md @@ -2,8 +2,8 @@ Two files. Import both. - OAN-dev-flow.postman_collection.json the requests - OAN-dev.postman_environment.json where your deployment's URLs go + api-collection.json the requests + local_postman_environment.json where your deployment's URLs go **The collection alone works against a tunnel.** Every URL variable defaults to loopback, because the stack publishes its ports on the VM's loopback only: @@ -30,7 +30,7 @@ Six requests, 32 assertions. Run a folder top to bottom the first time -- Publish seeds the catalogue Discover looks for -- and after that any request works on its own: - newman run OAN-dev-flow.postman_collection.json --folder "2. MandiPrice" + newman run api-collection.json --folder "2. MandiPrice" The two Select requests are the pair worth comparing. They hit the same endpoint on the same adapter and different domain packages answer them, because diff --git a/postman-collection/OAN-dev-flow.postman_collection.json b/postman-collection/api-collection.json similarity index 99% rename from postman-collection/OAN-dev-flow.postman_collection.json rename to postman-collection/api-collection.json index 91725bb..bf322c2 100644 --- a/postman-collection/OAN-dev-flow.postman_collection.json +++ b/postman-collection/api-collection.json @@ -1,7 +1,7 @@ { "info": { "_postman_id": "7458e3b4-dd16-4ec1-84ce-006f2e423183", - "name": "OAN dev \u2014 registry, publish, discover, select", + "name": "OAN API \u2014 publish, discover, select", "description": "The whole stack from Postman: the registry writes that set it up, updates for the fields that change, the reads that show what is there, and the three flows that use it.\n\nONE FOLDER PER CAPABILITY. Inside each, Publish seeds the catalogue Discover looks for, so run a folder top to bottom the first time; after that any request works on its own.\n\nTHERE ARE NO REGISTRY REQUESTS HERE, deliberately. The registry has no route through the gateway and publishes on loopback only, so nothing in a shared collection could reach it. bin/setup.py seeds all of it -- five participants and both capability bindings -- from the same .env the adapter configs are rendered from, which is what keeps the two from disagreeing.\n\nNOTHING HERE CHANGES THE STACK ON A DEFAULT RUN. The creates report \"already present\" once setup.py has run, because the registry is append-only. The updates write back the same values the variables already hold -- edit a variable to actually change a row.\n\nTHE UPDATES NEED A SCHEMA THAT PERMITS THEM. The registry re-validates the merged document on update, and that document carries its own osid and osUpdatedAt, so additionalProperties: false rejects the registry's own fields. Participant and ProviderSchema in config/registry/schemas/ must allow additional properties, and the registry reads schemas only at startup.\n\nEvery URL here is loopback, because the deployment publishes these ports on the VM's loopback only and nothing else is routable. Open a tunnel first and the defaults work unchanged:\n\n ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \\\n -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm\n\nNo VM hostname or address appears in this file.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "42114807" diff --git a/postman-collection/OAN-dev.postman_environment.json b/postman-collection/local_postman_environment.json similarity index 100% rename from postman-collection/OAN-dev.postman_environment.json rename to postman-collection/local_postman_environment.json diff --git a/quick-start/README.md b/quick-start/README.md index 73a59ea..2c858b4 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -672,10 +672,11 @@ Things worth knowing before editing any of this: ## Test it end to end -**Quickest path: import `../postman-collection/`.** Nineteen requests, 50 -assertions, nothing to fill in — the registry writes and reads that set the -stack up, then publish, discover and select for each capability, with every -value already matching this deployment. A green run means the stack is healthy +**Quickest path: import `../postman-collection/`.** Six requests, 32 +assertions, nothing to fill in — publish, discover and select for each +capability, with every value already matching this deployment. There are no +registry requests: the registry has no route through the edge, so `setup.py` +seeds it instead. A green run means the stack is healthy rather than merely answering. It sits at the repo root rather than in here, because it is not part of the From 7bab1fb0b93e6527c918276199ff7017bb6e8373 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 16:55:59 +0530 Subject: [PATCH 73/81] refactor: spell the experience adapter's config in full [OpenAgriNet/network-adapter#4] config/adapters/exp.yaml.tmpl -> experience.yaml.tmpl, and the rendered experience.yaml with it. The rename is only the filename, and that is the point. setup.py used one string for three different things -- the template's name, the key in keys/keys.json, and the __EXP_* placeholder prefix -- so renaming the role would have renamed the keys.json entry too, and setup.py would then have generated a FRESH keypair for a participant the registry has already published a public key for. This registry cannot update a record and its delete is soft, so the id could not be reused either: the adapter would sign with a key nobody can verify, and it would surface later as an authentication error with no obvious cause. setup.py documents that trap a few lines above, which is what made it worth honouring rather than discovering. So CONFIG_STEM maps the role to its filename and the role key stays "exp". Placeholders stay __EXP_*, keys/keys.json keeps its entry, and EXP_SUBSCRIBER_ID is untouched -- a deployed VM re-renders onto the new name and keeps its identity. Carried the new name into the compose mount, .gitignore, and the two places the README named the file. The compose SERVICE is still exp-adapter and the routing file is still routing-exp.yaml; renaming either is a separate change, the service because it is what NPM's proxy hosts resolve. --- quick-start/.gitignore | 2 +- quick-start/README.md | 4 ++-- quick-start/bin/setup.py | 24 +++++++++++++++---- .../{exp.yaml.tmpl => experience.yaml.tmpl} | 0 quick-start/docker-compose.yml | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) rename quick-start/config/adapters/{exp.yaml.tmpl => experience.yaml.tmpl} (100%) diff --git a/quick-start/.gitignore b/quick-start/.gitignore index d812134..015355a 100644 --- a/quick-start/.gitignore +++ b/quick-start/.gitignore @@ -6,7 +6,7 @@ keys/ # Rendered from the .tmpl files beside them, and they carry the private key # material that keys/ holds. The templates are the tracked source. -config/adapters/exp.yaml +config/adapters/experience.yaml config/adapters/network.yaml config/adapters/provider.yaml diff --git a/quick-start/README.md b/quick-start/README.md index 2c858b4..065cc83 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -931,7 +931,7 @@ config/ The routing table itself is not a file: it is rows in the npm-data volume. adapters/ - exp.yaml.tmpl templates. setup.py renders these to .yaml, + experience.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered provider.yaml.tmpl files hold private keys and are gitignored. routing-exp.yaml which action goes where. exp sends discover to @@ -1210,7 +1210,7 @@ that survives if you skip it. ```sh docker compose down -v # -v also deletes the registry and discovery data -rm -rf keys config/adapters/exp.yaml config/adapters/network.yaml config/adapters/provider.yaml +rm -rf keys config/adapters/experience.yaml config/adapters/network.yaml config/adapters/provider.yaml ``` Then start again from `docker compose up -d`. New keys mean new identities, so diff --git a/quick-start/bin/setup.py b/quick-start/bin/setup.py index 859bf68..e84938a 100755 --- a/quick-start/bin/setup.py +++ b/quick-start/bin/setup.py @@ -360,13 +360,29 @@ def key_osids(identities): # ------------------------------------------------------------------ configs +# The config filename for each role. +# +# Deliberately separate from the role key. That key is also the entry in +# keys/keys.json and the __EXP_* placeholder prefix, and changing it would make +# this script generate a FRESH keypair for a participant the registry has +# already published a public key for -- which it cannot update and whose delete +# is soft, so the id could not be reused either. The adapter would then sign +# with a key nobody can verify, and it would surface much later as an +# authentication error with no obvious cause. +# +# So the file can be spelled out in full without touching the thing that has to +# stay stable. +CONFIG_STEM = {"exp": "experience", "network": "network", "provider": "provider"} + + def render(identities): print("configs:") binding = f"{env('PROVIDER_PARTICIPANT_ID')}|{env('PROVIDER_CAPABILITY')}" mandi_binding = f"{env('MANDI_PARTICIPANT_ID')}|{env('MANDI_CAPABILITY')}" for role in ("exp", "network", "provider"): identity = identities[role] - template = (ADAPTERS / f"{role}.yaml.tmpl").read_text() + stem = CONFIG_STEM[role] + template = (ADAPTERS / f"{stem}.yaml.tmpl").read_text() prefix = role.upper() for placeholder, value in ( (f"__{prefix}_SUBSCRIBER_ID__", identity["participantId"]), @@ -386,8 +402,8 @@ def render(identities): ("__OTEL_ENVIRONMENT__", env("OTEL_ENVIRONMENT", "dev"))): template = template.replace(placeholder, value) if "__" in template: - sys.exit(f"setup: {role}.yaml still has unrendered placeholders") - out = ADAPTERS / f"{role}.yaml" + sys.exit(f"setup: {stem}.yaml still has unrendered placeholders") + out = ADAPTERS / f"{stem}.yaml" # A bare `docker compose up -d` before this script runs starts the # adapters too, and Docker creates a DIRECTORY at a bind-mount source # that does not exist. Writing would then fail with a bare @@ -402,7 +418,7 @@ def render(identities): f" make up") out.write_text(template) out.chmod(0o600) # holds a private key - print(f" config/adapters/{role}.yaml") + print(f" config/adapters/{stem}.yaml") if __name__ == "__main__": diff --git a/quick-start/config/adapters/exp.yaml.tmpl b/quick-start/config/adapters/experience.yaml.tmpl similarity index 100% rename from quick-start/config/adapters/exp.yaml.tmpl rename to quick-start/config/adapters/experience.yaml.tmpl diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index f01caa8..a472b78 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -451,7 +451,7 @@ services: <<: *adapter-env OTEL_SERVICE_NAME: oan-exp-adapter volumes: - - ./config/adapters/exp.yaml:/app/config/adapter.yaml:ro + - ./config/adapters/experience.yaml:/app/config/adapter.yaml:ro - ./config/adapters/routing-exp.yaml:/app/config/routing-exp.yaml:ro ports: - "127.0.0.1:${EXP_ADAPTER_PORT}:9202" From a62d746db1afd177c66b7556cc1888413f1bc7ae Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 17:07:23 +0530 Subject: [PATCH 74/81] docs: make the quick-start README a walkthrough, with the reference behind it [OpenAgriNet/network-adapter#4] It was 1218 lines with no entry point. The 299-line section on the public edge came BEFORE "Before you start", so the first thing a reader met was certificates and proxy hosts, and the commands that actually bring the stack up were three hundred lines down. About forty-five minutes of reading to find a ten-minute task. Now: a body of 375 lines, eight minutes, that is only the path through. Two parts, because they are two audiences -- Part 1 runs it on a laptop, Part 2 is the differences on a VM and nothing else, so neither reader wades through the other's material. Part 1 is eight numbered steps: prerequisites, configure, shared services, seed, then one step per layer, then verify. Each layer step has the same four headings -- what it does, the config it needs, bring it up, check it worked -- so the third one is skimmable once the first has been read. The steps run provider, network, experience, which is the reverse of the request flow and deliberately so: exp-adapter depends on the other two, so that is the order Compose starts them and the order the commands work in. The request flow is in the diagram at the top, where it belongs as orientation rather than as instructions. Everything else is Appendix A to O, moved VERBATIM. No reasoning was condensed away -- the edge, the registry contents, the troubleshooting, the request-flow detail, the startup-order argument are all intact, just no longer between a reader and their first command. The long-form "Before you start" and "Test it end to end" are kept as N and O rather than dropped, since the body supersedes their steps but not their explanation. Two errors of mine found while writing it, both by checking rather than assuming: - APP_NETWORK_ID was listed as the network adapter's config. It is the discovery service's -- the network template never references it. Moved to the step where it is actually read. - The provider step listed participant ids without their capabilities. It is the PAIR that makes a binding key, which is what makes a mismatch a 404 rather than a config error, so both halves are named now. Verified: no dangling appendix references, code fences balanced, both tables render, no reference left pointing at an old section title. --- quick-start/README.md | 1120 +++++++++++++++++++++++++++-------------- 1 file changed, 743 insertions(+), 377 deletions(-) diff --git a/quick-start/README.md b/quick-start/README.md index 065cc83..de696d5 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -1,15 +1,381 @@ -# OAN stack, on Docker Compose +# OAN quick-start -The whole OpenAgriNet stack for a **dev deployment on a VM**: the registry, the -discovery service, the three adapters, and a mock upstream per capability so a -request has something to answer it. One compose file, one config folder, -`make up`. +The whole OAN stack in Docker Compose: a registry, a discovery service, three +adapters, and two mock upstreams standing in for real provider APIs. -This is a dev environment. It is not production: the adapter signing keys sit -in a config file on disk, nothing terminates TLS, and every credential shipped -in `.env.example` is a public default. +Nothing is built here — the images are pulled. Locally that is about ten +minutes, most of it waiting for Keycloak. -## What is here, and what is not +## The stack in one picture + +``` + consumer + │ + ▼ + ┌──────────── experience adapter ────────────┐ the caller's edge + │ │ + │ discover ──► network adapter ──► discovery service + │ │ + │ select ────────────────────────► provider adapter + └────────────────────────────────────────────┘ │ + ▼ + mockimd · mockagmarknet + + every adapter reads the registry: who signed this, and where does + this capability's provider live +``` + +A `discover` asks the network layer what exists. A `select` goes straight to +the provider layer, which calls the upstream and answers in the same HTTP +round trip — there is no callback. + +That is the request order. The **startup** order is the reverse: the +experience adapter depends on the other two, so Compose brings them up first. +The steps below are in startup order, so they work top to bottom. + +## Which path are you on? + +| | Local | VM | +|----------------------|--------------------------|-----------------------------| +| Reached over | `localhost` | a public hostname, TLS | +| Public edge (Nginx) | not started | started, on 80 and 443 | +| Observability | not started | optional, wants 2–4 GB | +| Credentials | shipped defaults are ok | **must all be changed** | +| Command | `make up-core` | `make up` | + +**Local** is Part 1. **VM** is Part 1, then Part 2 for the differences. + +--- + +# Part 1 — Run it locally + +## Step 1 — Prerequisites + +- Docker with Compose v2 — `docker compose version` must work, not `docker-compose` +- `python3`, and the `cryptography` package: `pip install cryptography` +- `curl` + +`make up` checks all of these before starting anything, because Keycloak's +healthcheck can take five minutes on a cold volume and a missing dependency +should not surface after that wait. + +## Step 2 — Configure + +```sh +cp .env.example .env +``` + +Locally you need change nothing. Read the file once anyway — it is commented +where the reasoning is not obvious, and it is the reference for every key. + +Two things to know: + +**The ports.** Published on `localhost` only. + +``` +8081 registry 9200 provider adapter 9100 mockimd +8080 keycloak 9201 network adapter 9101 mockagmarknet +9990 keycloak admin 9202 experience adapter +8090 discovery +``` + +If any of those is already taken, change it in `.env` — that is the only edit +a local run needs. The adapters reach each other by Compose service name, not +through the published ports, so moving them changes only what you type into +Postman. + +**The images.** `ADAPTER_IMAGE` and the adapter configs in this repo move +together: the configs name plugins by id, and an id is the basename of a `.so` +inside the image. A mismatch is `unrecognized step: ` at startup, which +reads like a config typo and is not. + +## Step 3 — Start the shared services + +Everything else depends on these. + +```sh +docker compose up -d registry discovery +``` + +Keycloak and both databases come up with them. + +One key in `.env` belongs to this step: `APP_NETWORK_ID` is the network every +published catalogue is filed under, and the discovery service reads it. It is +also what a `discover` filters on, so a request naming a different network +finds nothing. + +Wait for the registry to report healthy — up to five minutes the first time, +seconds after that: + +```sh +docker compose ps registry +``` + +## Step 4 — Seed and render + +```sh +python3 bin/setup.py +``` + +One idempotent script, three jobs: + +- generates a keypair per adapter into `keys/keys.json` +- registers **five participants** — three adapters, two upstreams — and **two + capability bindings** +- renders `config/adapters/{experience,network,provider}.yaml` from the + `.tmpl` files beside them + +It reads the same `.env` that seeds the registry, which is what keeps the +registry rows and the adapter configs from disagreeing. + +Two things it is worth knowing now rather than later. It **must** run before +any adapter starts: an adapter's config is a bind-mounted *file*, and Docker +creates a *directory* at any bind-mount source that does not exist. And +`keys/keys.json` is the only copy of those keypairs — the registry cannot +update a published key, so losing the file means picking new participant ids. + +→ Appendix D for what it wrote and how to look at it. + +## Step 5 — Provider layer + +Calls the upstreams and answers `select`. The only layer that talks to a +provider, and it serves both capabilities from one adapter. + +**Config it needs**, in `.env`: + +``` +PROVIDER_SUBSCRIBER_ID this adapter's own network identity +PROVIDER_PARTICIPANT_ID the weather upstream's id ─┐ each pairs with its +PROVIDER_CAPABILITY openagrinet:WeatherObservation │ *_CAPABILITY to make +MANDI_PARTICIPANT_ID the mandi upstream's id ─┤ a binding key, which +MANDI_CAPABILITY openagrinet:MandiPrice ┘ is how a step knows +MANDI_TOKEN the mandi upstream's credential its own work +``` + +Those same four values seed the registry rows, which is why changing one here +means re-running `setup.py` — and why a mismatch shows up as a 404 rather than +a config error. → Appendix F. + +`setup.py` renders these into `config/adapters/provider.yaml`. Do not edit +that file — it is regenerated, and it holds a private key. + +**Bring it up** + +```sh +docker compose up -d provider-adapter +``` + +Compose starts the registry and both mocks first; it will not come up without +them. + +**Check it worked** + +```sh +docker compose logs provider-adapter | grep 'Processor steps initialized' +``` + +You want the capability steps listed by name. + +## Step 6 — Network layer + +Fronts discovery. Verifies the caller, passes `discover` and `publish` on to +the discovery service, and re-signs as itself. + +**Config it needs** + +``` +NETWORK_SUBSCRIBER_ID this adapter's own network identity +``` + +**Bring it up** + +```sh +docker compose up -d network-adapter +``` + +**Check it worked** + +```sh +docker compose logs network-adapter | grep 'Server listening' +``` + +## Step 7 — Experience layer + +The consumer's edge. Sends `discover` to the network layer and `select` +straight to the provider layer — which action goes where is +`config/adapters/routing-exp.yaml`, not a code path. + +**Config it needs** + +``` +EXP_SUBSCRIBER_ID this adapter's own network identity +``` + +**Bring it up** + +```sh +docker compose up -d exp-adapter +``` + +**Check it worked** + +```sh +make ps +``` + +All three adapters running. Or bring the whole thing up in one command, in the +order it has to happen: + +```sh +make up-core +``` + +## Step 8 — Verify end to end + +Import both files from `../postman-collection/` into Postman: + +``` +api-collection.json the requests +local_postman_environment.json the URLs, already pointing at localhost +``` + +Run it top to bottom: **6 requests, 32 assertions**, two folders, one per +capability. Each folder publishes a catalogue, discovers it, then selects +against it — so run publish before discover the first time. + +A green run means the registry is seeded, both adapters sign and verify, both +mappings work, and discovery is indexing. + +There are no registry requests in the collection deliberately — the registry +has no route through the edge, so `setup.py` seeds it instead. + +--- + +# Part 2 — Run it on a VM + +Part 1 Steps 2–8 apply as written. This is only what is different. + +## Step V1 — Prepare the VM + +```sh +bin/bootstrap-ubuntu.sh +``` + +Docker, Compose v2, `python3-cryptography` and the docker group, idempotent. +It deliberately does not clone anything, write `.env` or start anything — +those need decisions that do not belong in a script piped from the internet. + +**8 GB** runs the stack. **16 GB** if you want the observability tier, which +ClickHouse alone can spend 2–4 GB on. + +## Step V2 — Change every credential + +`.env.example` ships working defaults, which means they are public. Change all +of them before the VM is reachable by anyone but you: + +``` +POSTGRES_PASSWORD KEYCLOAK_ADMIN_PASSWORD KEYCLOAK_SECRET +REGISTRY_DEFAULT_USER_PASSWORD +``` + +The adapter keypairs are the exception — `setup.py` generates those, and they +are never written to `.env`. + +## Step V3 — Bring it up + +```sh +make up +``` + +`make up` rather than `make up-core`: two more tiers. + +``` +4. nginx-proxy-manager the public edge — 80 and 443, all interfaces +5. hyperdx ClickStack. Optional, and the reason for 16 GB. +``` + +Step 4 is the one that makes the VM reachable from the internet. + +## Step V4 — Expose it, and decide what is exposed + +Every port except the edge's 80 and 443 is bound to `127.0.0.1`, written +literally in `docker-compose.yml` rather than taken from a variable. One +switch that moves every port to the public interface at once is a footgun; the +ports that should be reachable are reachable through the edge instead. + +So the registry, Keycloak and the databases are **not** publicly reachable, +and that is deliberate — a registry whose write token any reader of `.env.example` +can mint should not be on the internet. + +Adding the proxy hosts, requesting certificates, and the `/publish` deny that +every host gets → **Appendix B**. Read it before pointing DNS at the box. + +## Step V5 — Reach the loopback ports + +From a workstation: + +```sh +ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ + -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm +``` + +The collection's defaults then work unchanged, because they already point at +loopback. + +## Step V6 — Observability (optional) + +```sh +make observability +``` + +Three signals over OTLP/gRPC to HyperDX. `OTEL_ENABLED=false` builds no +exporter at all, which is what you want on a box with no collector — leaving +it true against a missing one is the noisy case. → Appendix H. + +--- + +## If something is wrong + +- **`unrecognized step: `** — `ADAPTER_IMAGE` predates the config. + Appendix G. +- **404 `NET_ENTITY_NOT_FOUND`** — no provider step claimed the payload; a + binding key disagrees with `.env`. Appendix F. +- **404 naming a binding with no active record** — the step matched but the + registry has no `ProviderSchema` row for it. Appendix F. +- **502 from a `select`** — the upstream answered non-2xx. Appendix F. +- **NPM's default page, or a 502 that worked yesterday** — Appendix B. +- **An adapter config is a directory** — something started before + `setup.py`. Appendix F. + +Full set, with what to run for each → **Appendix F**. + +--- + +## Appendices + +Reference, read on demand. Nothing here is a step. + +| | | +|---|---| +| **A** | What is here, and what is not | +| **B** | Reaching it: the edge, routes, certificates, tunnels | +| **C** | Startup order, and why it is that order | +| **D** | What is in the registry, and why you did not create it | +| **E** | How a request flows | +| **F** | When it does not work | +| **G** | Updating a deployment that is already running | +| **H** | Telemetry | +| **I** | Schema validation | +| **J** | About the mapping files | +| **K** | The layout | +| **L** | Renaming this directory | +| **M** | Starting over | +| **N** | Before you start, and bringing it up — the long form | +| **O** | Testing it end to end — the long form | + +--- + +## Appendix A — What is here, and what is not Running here: @@ -49,7 +415,7 @@ Deliberately **not** here: The provider adapter never holds that address in a config file; it reads it from the registry per request. -## Reaching it +## Appendix B — Reaching it Two doors, and which one you use depends on what you are reaching. @@ -348,24 +714,7 @@ There is no `BIND_ADDR` any more. It used to move every published port onto the public interface at once, which is a footgun once something exists to expose the one tier that should be reachable. -## Before you start - -On the VM: - -- Docker with Compose **v2.24 or newer**, logged in to wherever the images live - if it is private — `docker login ghcr.io`. The version floor is the - `env_file: required: false` on the HyperDX service, which is what lets an - absent `.env.docker` be absent instead of fatal. -- Python 3 and the `cryptography` package — `pip install cryptography` -- 16 GB of RAM if you run the `observability` profile — ClickHouse alone wants - 2-4 GB on top of the two JVM services. 8 GB is workable without it. - -Nothing else. No external API and no tunnel: the two mock upstreams are part -of the stack, so a select has something to answer it the moment it comes up. - -`bin/bootstrap-ubuntu.sh` installs the first two on a fresh Ubuntu VM. - -## Bring it up +## Appendix C — Startup order, and why it is that order ```sh cp .env.example .env @@ -443,85 +792,7 @@ That 403 is the check worth repeating after any NPM change: it is the only evidence that `npm-custom/server_proxy.conf` is still mounted, and losing the mount silently opens an unauthenticated catalogue write. -## Updating a deployment that is already running - -Three things can change, and they need different work. Getting this wrong is -the most likely way to break a working VM, so the order matters. - -**Config only** — a `.tmpl`, a routing file, `.env`. Re-render and recreate: - -```sh -make pull # git pull, and fixes the ownership NPM leaves behind -make up # step 2 re-renders the adapter configs, then recreates -``` - -`make restart` is not enough on its own for a `.tmpl` change: the adapters read -a rendered `.yaml`, and only `setup.py` writes it. - -**A new adapter image as well.** Any change to the plugin ids in -`config/adapters/*.tmpl` is this case, because an id is the basename of a `.so` -inside the image. The new config must not meet the old image, or every adapter -dies at startup. - -If `ADAPTER_IMAGE` names a **new tag**, set it before `make up` and that is -all. If it follows **`latest`**, `make up` alone is not enough: `pull_policy: -missing` means a tag already on disk is never re-fetched, and nothing in -`stack.sh` pulls, so the stack would quietly come back on the old image. Fetch -it explicitly first: - -```sh -docker compose pull provider-adapter network-adapter exp-adapter -``` - -Either way, build it from the adapter repo at the commit the config expects: - -```sh -git clone https://github.com/OpenAgriNet/network-adapter.git -cd network-adapter && git checkout - -docker build -f Dockerfile.adapter-with-plugins \ - --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ - -t ghcr.io//oan-adapter:$(git rev-parse --short HEAD) . - -# the check worth doing before you push or deploy it -docker run --rm --entrypoint sh ghcr.io//oan-adapter: \ - -c 'ls plugins/ | grep -iE "weather|mandi"' -``` - -That last command should print the ids the config actually names. If it prints -something else, the image is from the wrong commit and nothing downstream will -work. - -**Payload shapes changed.** If `@context` moved, catalogues already in the -discovery database still carry the old value, and `discover` matches -`schemaContext` by exact string equality — so discover alone returns zero rows -against a database seeded before the change. Run the collection top to bottom -so publish reseeds first. `updateMode: MERGE` on the same `catalogId` updates -in place rather than duplicating. - -**Then check, in this order.** Cheapest first, because each failure explains -the next: - -```sh -docker compose logs provider-adapter | grep 'Processor steps initialized' -docker compose logs provider-adapter | grep -iE '"level":"(error|fatal)"' -make ps -``` - -The first should list the capability steps by the ids the config names. The -second should be empty. Only then run the collection. - -**Rolling back** is `git checkout `, `ADAPTER_IMAGE` back to the -old image, `make up`. Both, together — the old image with the new config fails -at startup, and the new image with the old config starts but silently runs the -old behaviour. - -Note this is the case `latest` serves badly. Rolling the config back is exact, -but "the old image" has no name if the tag has already moved, so you would be -recovering it by digest — `docker images --digests` on the VM, if it is still -there at all. Pin a tag before a change you might need to undo. - -## What is in the registry, and why you did not create it +## Appendix D — What is in the registry, and why you did not create it `bin/setup.py` wrote all of it. Nothing in this section is a step to perform — it is what to look at when something does not match. @@ -670,118 +941,9 @@ Things worth knowing before editing any of this: `config/registry/schemas/` needs `docker compose restart registry` before it takes effect. -## Test it end to end +## Appendix E — How a request flows -**Quickest path: import `../postman-collection/`.** Six requests, 32 -assertions, nothing to fill in — publish, discover and select for each -capability, with every value already matching this deployment. There are no -registry requests: the registry has no route through the edge, so `setup.py` -seeds it instead. A green run means the stack is healthy -rather than merely answering. - -It sits at the repo root rather than in here, because it is not part of the -compose stack — it is what you point at one, and its environment file exists so -it can be aimed somewhere else. - -The rest of this section is one of those requests as curl, if you would rather -see it than run it. - -```sh -curl -s -X POST http://127.0.0.1:9202/select \ - -H 'Content-Type: application/json' \ - -d '{ - "context": { - "version": "2.0.0", "action": "select", - "networkId": "oan-dev", - "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", - "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", - "timestamp": "2026-09-04T06:12:01.330Z" - }, - "message": { "contract": { "commitments": [ { - "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, - "resources": [ { - "id": "res:mausamgram:point-forecast", - "quantity": 1, - "resourceAttributes": { - "@context": "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld", - "@type": "openagrinet:WeatherObservation", - "subjectCategories": ["Weather"], - "informationMode": "OnDemand", - "supportedObservationTypes": ["Forecast"], - "supportedParameters": ["Rainfall", "Temperature"], - "geographicGranularities": ["Point"], - "location": { "type": "Point", "coordinates": [73.7898, 19.9975] } - } - } ], - "offer": { - "id": "offer:mausamgram:open-data", - "resourceIds": ["res:mausamgram:point-forecast"], - "provider": { "id": "mausamgram-mock", - "descriptor": { "code": "IMD-NWP-01", "name": "IMD Mausamgram NWP" } } - } - } ] } } - }' | python3 -m json.tool -``` - -An `on_select` comes back with one resource per forecast day — three by -default, which is `MOCKIMD_DAYS`. - -The mandi equivalent is the same call to the same endpoint with a `MandiPrice` -resource and `agmarknet-mock` as the provider, and that is the point worth -taking from this section: **one endpoint, two capabilities, and no routing -config in between.** Each provider step builds a binding key out of the -payload it is handed, answers if the key is its own, and passes the payload -through untouched if it is not. Adding a third capability is a plugin and two -registry rows, not a new route. - -Two things about the payload: - -**No party is named, in either direction.** Identity travels in the -`Authorization` header's `keyId`, which names the signer and the key the -registry published for it; a body that declares no caller simply skips the -declared-identity comparison. Nothing needs `bapId` or `bppId`, and the `*Uri` -fields they came with were container-internal addresses that meant nothing -outside this compose network anyway. - -**The experience adapter is the only one that takes an unsigned request.** The -experience app is inside the trust boundary, so there is no network signature -to check — which is what makes this testable with a plain curl. The same call -to the provider adapter on 9200 is rejected unsigned. - -## Telemetry - -`docker compose --profile observability up -d` brings up HyperDX on -`127.0.0.1:8085` (tunnel to reach it) with OTLP on 4317/4318. It is -`clickstack-local`, not `clickstack-all-in-one`: local runs single-user with no -team to create and no ingestion key to mint, which is what makes `up -d` the -whole setup step — and also why it must stay on loopback, since there is no -login in front of it. - -**What actually arrives today is less than the wiring suggests, and that is -worth knowing before you go looking for traces that are not there.** - -- **discovery** reads `OTEL_EXPORTER` and `OTEL_EXPORTER_OTLP_ENDPOINT` into - its config, and nothing in the current build consumes them — the only - OpenTelemetry packages in its `go.mod` are indirect. So `OTEL_EXPORTER` - stays `none` by default; setting it to `otlp` emits nothing rather than - failing. When the exporter is wired, `OTEL_EXPORTER=otlp` in `.env` is the - whole change and the endpoint already points here. -- **the three adapters** get `OTEL_EXPORTER_OTLP_ENDPOINT` and - `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`. Whether that image's SDK reads - them is unverified in either direction — the image is pulled and its source - is not in this repo. Nothing depends on the answer: an absent collector makes - an OTLP exporter drop spans, not fail a request. -- **container logs go nowhere near HyperDX** without something to ship them. - `docker compose logs -f ` remains the way to read them. Shipping - them would mean an OTel collector with a `filelog` receiver over - `/var/lib/docker/containers`, which is not in this stack. - -So treat this profile as the destination being ready and in one place, rather -than as observability that is switched on. - -## How a request flows - -Three paths, and which adapter answers is the whole design: +Three paths, and which adapter answers is the whole design: ``` discover you -> exp -> network -> discovery service @@ -849,158 +1011,7 @@ Three things about that: signature — and its identity check skips a body that declares no caller rather than demanding one. -## Schema validation - -Every adapter loads the pinned Beckn v2 LTS spec and validates request bodies -against it. On the **provider adapter** a second layer runs too: it walks the -payload for objects carrying `@context` and `@type`, resolves the schema that -`@type` names, and validates the object against it. Base validation treats -`resourceAttributes` as a free-form object, so this is the only layer that -checks a capability's own attributes at all. - -The schemas are not in this repository and are not mounted. Each resource's -`@context` names the published pack, and the validator swaps `context.jsonld` -for `attributes.yaml` to fetch the schema beside it: - -``` -@context .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld -fetched .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml -``` - -So a payload names the pack revision it wants to be judged against, and there is -no copy here to drift from the published one — the same reason the mappings are -fetched rather than committed. - -It is cached for 24 hours, so only the first payload after a restart pays for -the fetch. Two consequences: the provider adapter needs egress to -`raw.githubusercontent.com`, and a fetch that **fails rejects the payload** -rather than skipping validation. An `@context` on any other host is refused -before anything is fetched — `extendedSchema_allowedDomains` in the config is -the list. - -**What it does not check.** The validator library parses `if`/`then`/`else` but -never evaluates it, so every pack rule predicated on `informationMode` is -unenforced — a pass here is not full conformance to a pack. It does enforce -types, string formats, `enum`, `const`, `required`, `additionalProperties`, -`not` and `allOf`/`anyOf`/`oneOf`. - -Three consequences worth knowing before you write a payload: - -- **Each resource under a commitment needs a `quantity`.** The spec's - `Commitment.resources` requires `["id", "quantity"]` while `Resource` itself - defines no `quantity` property and the spec has no `Quantity` schema at all — - a defect upstream, not something this deployment chose. Any value satisfies - it. Without one, every `select` is refused with - `SCH_REQUIRED_FIELD_MISSING: property "quantity" is missing`. -- **A `date-time` field will not take a bare date.** `validity.startsAt` and - `endsAt` are `format: date-time` in the packs, so `2025-08-20` is refused - and `2025-08-20T00:00:00+05:30` is accepted. `arrivalDate` is `format: date` - and wants the opposite. -- **`publish` is validated, on the provider adapter.** Declaring the validator - is not enough — a plugin missing from `steps:` never runs, which is why - publish went unchecked for a while — so `validateSchema` is in that module's - `steps:` and its resources are checked against their packs like any other. - The network adapter validates nothing: its single module runs - `validateSign`, `addRoute`, `sign` and never declares a validator. - -An action the spec does not know, or a body missing a required field, comes -back as a signed NACK with a `SCH_*` code and the JSON path that failed. - -## The layout - -``` -docker-compose.yml the whole stack. Read it in tiers -- the banner - comments are the structure: registry, discovery, - adapters, observability (profile), edge (profile) -.env.example copy to .env -Makefile the front door: make up / up-core / down / help. -bin/ - bootstrap-ubuntu.sh docker and python on a fresh Ubuntu VM - stack.sh the startup order, and why it is that order. - Every make target is one line of delegation here. - setup.py keys, five registry rows, the adapter configs -config/ - reverse-proxy/ - npm-custom/ mounted to /data/nginx/custom, which NPM includes - http_top.conf on its own: the rate-limit zone declaration, - server_proxy.conf and the /publish deny that every proxy host gets - npm-advanced/ - exp.conf NOT loaded -- paste into the experience host's - Advanced tab. Kept here because a textarea in - NPM's database is not reviewable. - The routing table itself is not a file: it is - rows in the npm-data volume. - adapters/ - experience.yaml.tmpl templates. setup.py renders these to .yaml, - network.yaml.tmpl filling in the keys it generated. The rendered - provider.yaml.tmpl files hold private keys and are gitignored. - routing-exp.yaml which action goes where. exp sends discover to - routing-network.yaml the network layer and select to the provider; - routing-provider.yaml provider sends publish to the network layer; - network sends discover and publish to discovery - registry/ - schemas/ Participant, ProviderSchema, SchemaRegistry. - Read at startup -- a change needs the registry - service restarted. - imports/ the Keycloak realm - discovery/ - instance.yaml.example optional override; see the compose file - mappings/ - mausamgram/ one file per binding-action: the request and the - agmarknet/ response transformation, in JSONata. These are the - files the adapters fetch over the raw CDN -- the - served copy and the reviewable copy are one file -mock-server/ - mockimd/ the two mock upstreams. Sources only: they are - mockagmarknet/ pulled as published images like everything else. - See mock-server/README.md for the build commands and - for what each deliberately gets wrong. -../postman-collection/ NOT in here -- a sibling of this directory. The - collection plus an environment file, because it is - what you point AT a stack rather than part of one -keys/keys.json generated, gitignored. The private halves of the - three adapter keypairs -- the one file here that - is worth backing up, and the reason setup.py can - be re-run without invalidating what it registered -``` - -## About the mapping files - -`config/mappings/` holds the two this deployment uses — one per binding-action -— and `MAPPING_URL` and `MANDI_MAPPING_URL` point at **this repo's own copies** -over GitHub's raw CDN. So the file a reader reviews and the file the adapter -fetches are one file, and cannot drift. - -Each file has two halves. The request half turns the incoming Beckn payload -into the query string or body the upstream expects; the response half turns -what comes back into the resources that go in the answer. The mandi one is the -better example of why this is not a field-renaming exercise: it converts ISO -dates to the `dd-MM-yyyy` Agmarknet wants, sends `marketcode` only when the -request carried one, turns price strings into numbers, and omits a price that -was not reported rather than sending a zero. - -It is a URL rather than a path because the registry publishes the full URL and -the adapter fetches it verbatim — which means a mapping has to be reachable -before it can be tested, and what this stack exercises is exactly what any -consumer fetches. - -Note the branch in those URLs. Once this merges, point them at the default -branch, or pin a tag so a deployment is not following a moving file. - -**What can be fixed here without touching code.** Quite a lot, and this is the -design intent: when a real upstream turns out to answer with different field -names, a different date format, or a nested envelope, that is a mapping edit -and a cache expiry. What is *not* fixable here is anything that depends on the -response never arriving — a non-2xx never reaches the mapping, because the -step fails first. - -To change one: edit the file here and push, or publish a fork anywhere that -serves raw text over https and put that URL in the `mappings` field of the -ProviderSchema row. The adapter caches a mapping for `cacheTTL` (one minute, -in the adapter config) and GitHub's raw CDN caches for about five, so give an -edit a few minutes to show up. - -## When it does not work +## Appendix F — When it does not work Both of the common failures are a binding key disagreeing with itself, and which 404 you get says which side is wrong. @@ -1158,14 +1169,274 @@ docker run --rm -v quick-start_npm-data:/data -v "$PWD":/backup \ alpine tar czf /backup/npm-data.tgz -C /data . ``` -## Renaming this directory +## Appendix G — Updating a deployment that is already running -Worth knowing before you pull a rename onto a running host, because Docker will -not warn you. +Three things can change, and they need different work. Getting this wrong is +the most likely way to break a working VM, so the order matters. -**Compose takes its project name from the directory holding the compose file**, -and every named volume is prefixed with it. So this directory becoming -`quick-start` renames all five: +**Config only** — a `.tmpl`, a routing file, `.env`. Re-render and recreate: + +```sh +make pull # git pull, and fixes the ownership NPM leaves behind +make up # step 2 re-renders the adapter configs, then recreates +``` + +`make restart` is not enough on its own for a `.tmpl` change: the adapters read +a rendered `.yaml`, and only `setup.py` writes it. + +**A new adapter image as well.** Any change to the plugin ids in +`config/adapters/*.tmpl` is this case, because an id is the basename of a `.so` +inside the image. The new config must not meet the old image, or every adapter +dies at startup. + +If `ADAPTER_IMAGE` names a **new tag**, set it before `make up` and that is +all. If it follows **`latest`**, `make up` alone is not enough: `pull_policy: +missing` means a tag already on disk is never re-fetched, and nothing in +`stack.sh` pulls, so the stack would quietly come back on the old image. Fetch +it explicitly first: + +```sh +docker compose pull provider-adapter network-adapter exp-adapter +``` + +Either way, build it from the adapter repo at the commit the config expects: + +```sh +git clone https://github.com/OpenAgriNet/network-adapter.git +cd network-adapter && git checkout + +docker build -f Dockerfile.adapter-with-plugins \ + --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ + -t ghcr.io//oan-adapter:$(git rev-parse --short HEAD) . + +# the check worth doing before you push or deploy it +docker run --rm --entrypoint sh ghcr.io//oan-adapter: \ + -c 'ls plugins/ | grep -iE "weather|mandi"' +``` + +That last command should print the ids the config actually names. If it prints +something else, the image is from the wrong commit and nothing downstream will +work. + +**Payload shapes changed.** If `@context` moved, catalogues already in the +discovery database still carry the old value, and `discover` matches +`schemaContext` by exact string equality — so discover alone returns zero rows +against a database seeded before the change. Run the collection top to bottom +so publish reseeds first. `updateMode: MERGE` on the same `catalogId` updates +in place rather than duplicating. + +**Then check, in this order.** Cheapest first, because each failure explains +the next: + +```sh +docker compose logs provider-adapter | grep 'Processor steps initialized' +docker compose logs provider-adapter | grep -iE '"level":"(error|fatal)"' +make ps +``` + +The first should list the capability steps by the ids the config names. The +second should be empty. Only then run the collection. + +**Rolling back** is `git checkout `, `ADAPTER_IMAGE` back to the +old image, `make up`. Both, together — the old image with the new config fails +at startup, and the new image with the old config starts but silently runs the +old behaviour. + +Note this is the case `latest` serves badly. Rolling the config back is exact, +but "the old image" has no name if the tag has already moved, so you would be +recovering it by digest — `docker images --digests` on the VM, if it is still +there at all. Pin a tag before a change you might need to undo. + +## Appendix H — Telemetry + +`docker compose --profile observability up -d` brings up HyperDX on +`127.0.0.1:8085` (tunnel to reach it) with OTLP on 4317/4318. It is +`clickstack-local`, not `clickstack-all-in-one`: local runs single-user with no +team to create and no ingestion key to mint, which is what makes `up -d` the +whole setup step — and also why it must stay on loopback, since there is no +login in front of it. + +**What actually arrives today is less than the wiring suggests, and that is +worth knowing before you go looking for traces that are not there.** + +- **discovery** reads `OTEL_EXPORTER` and `OTEL_EXPORTER_OTLP_ENDPOINT` into + its config, and nothing in the current build consumes them — the only + OpenTelemetry packages in its `go.mod` are indirect. So `OTEL_EXPORTER` + stays `none` by default; setting it to `otlp` emits nothing rather than + failing. When the exporter is wired, `OTEL_EXPORTER=otlp` in `.env` is the + whole change and the endpoint already points here. +- **the three adapters** get `OTEL_EXPORTER_OTLP_ENDPOINT` and + `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`. Whether that image's SDK reads + them is unverified in either direction — the image is pulled and its source + is not in this repo. Nothing depends on the answer: an absent collector makes + an OTLP exporter drop spans, not fail a request. +- **container logs go nowhere near HyperDX** without something to ship them. + `docker compose logs -f ` remains the way to read them. Shipping + them would mean an OTel collector with a `filelog` receiver over + `/var/lib/docker/containers`, which is not in this stack. + +So treat this profile as the destination being ready and in one place, rather +than as observability that is switched on. + +## Appendix I — Schema validation + +Every adapter loads the pinned Beckn v2 LTS spec and validates request bodies +against it. On the **provider adapter** a second layer runs too: it walks the +payload for objects carrying `@context` and `@type`, resolves the schema that +`@type` names, and validates the object against it. Base validation treats +`resourceAttributes` as a free-form object, so this is the only layer that +checks a capability's own attributes at all. + +The schemas are not in this repository and are not mounted. Each resource's +`@context` names the published pack, and the validator swaps `context.jsonld` +for `attributes.yaml` to fetch the schema beside it: + +``` +@context .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld +fetched .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml +``` + +So a payload names the pack revision it wants to be judged against, and there is +no copy here to drift from the published one — the same reason the mappings are +fetched rather than committed. + +It is cached for 24 hours, so only the first payload after a restart pays for +the fetch. Two consequences: the provider adapter needs egress to +`raw.githubusercontent.com`, and a fetch that **fails rejects the payload** +rather than skipping validation. An `@context` on any other host is refused +before anything is fetched — `extendedSchema_allowedDomains` in the config is +the list. + +**What it does not check.** The validator library parses `if`/`then`/`else` but +never evaluates it, so every pack rule predicated on `informationMode` is +unenforced — a pass here is not full conformance to a pack. It does enforce +types, string formats, `enum`, `const`, `required`, `additionalProperties`, +`not` and `allOf`/`anyOf`/`oneOf`. + +Three consequences worth knowing before you write a payload: + +- **Each resource under a commitment needs a `quantity`.** The spec's + `Commitment.resources` requires `["id", "quantity"]` while `Resource` itself + defines no `quantity` property and the spec has no `Quantity` schema at all — + a defect upstream, not something this deployment chose. Any value satisfies + it. Without one, every `select` is refused with + `SCH_REQUIRED_FIELD_MISSING: property "quantity" is missing`. +- **A `date-time` field will not take a bare date.** `validity.startsAt` and + `endsAt` are `format: date-time` in the packs, so `2025-08-20` is refused + and `2025-08-20T00:00:00+05:30` is accepted. `arrivalDate` is `format: date` + and wants the opposite. +- **`publish` is validated, on the provider adapter.** Declaring the validator + is not enough — a plugin missing from `steps:` never runs, which is why + publish went unchecked for a while — so `validateSchema` is in that module's + `steps:` and its resources are checked against their packs like any other. + The network adapter validates nothing: its single module runs + `validateSign`, `addRoute`, `sign` and never declares a validator. + +An action the spec does not know, or a body missing a required field, comes +back as a signed NACK with a `SCH_*` code and the JSON path that failed. + +## Appendix J — About the mapping files + +`config/mappings/` holds the two this deployment uses — one per binding-action +— and `MAPPING_URL` and `MANDI_MAPPING_URL` point at **this repo's own copies** +over GitHub's raw CDN. So the file a reader reviews and the file the adapter +fetches are one file, and cannot drift. + +Each file has two halves. The request half turns the incoming Beckn payload +into the query string or body the upstream expects; the response half turns +what comes back into the resources that go in the answer. The mandi one is the +better example of why this is not a field-renaming exercise: it converts ISO +dates to the `dd-MM-yyyy` Agmarknet wants, sends `marketcode` only when the +request carried one, turns price strings into numbers, and omits a price that +was not reported rather than sending a zero. + +It is a URL rather than a path because the registry publishes the full URL and +the adapter fetches it verbatim — which means a mapping has to be reachable +before it can be tested, and what this stack exercises is exactly what any +consumer fetches. + +Note the branch in those URLs. Once this merges, point them at the default +branch, or pin a tag so a deployment is not following a moving file. + +**What can be fixed here without touching code.** Quite a lot, and this is the +design intent: when a real upstream turns out to answer with different field +names, a different date format, or a nested envelope, that is a mapping edit +and a cache expiry. What is *not* fixable here is anything that depends on the +response never arriving — a non-2xx never reaches the mapping, because the +step fails first. + +To change one: edit the file here and push, or publish a fork anywhere that +serves raw text over https and put that URL in the `mappings` field of the +ProviderSchema row. The adapter caches a mapping for `cacheTTL` (one minute, +in the adapter config) and GitHub's raw CDN caches for about five, so give an +edit a few minutes to show up. + +## Appendix K — The layout + +``` +docker-compose.yml the whole stack. Read it in tiers -- the banner + comments are the structure: registry, discovery, + adapters, observability (profile), edge (profile) +.env.example copy to .env +Makefile the front door: make up / up-core / down / help. +bin/ + bootstrap-ubuntu.sh docker and python on a fresh Ubuntu VM + stack.sh the startup order, and why it is that order. + Every make target is one line of delegation here. + setup.py keys, five registry rows, the adapter configs +config/ + reverse-proxy/ + npm-custom/ mounted to /data/nginx/custom, which NPM includes + http_top.conf on its own: the rate-limit zone declaration, + server_proxy.conf and the /publish deny that every proxy host gets + npm-advanced/ + exp.conf NOT loaded -- paste into the experience host's + Advanced tab. Kept here because a textarea in + NPM's database is not reviewable. + The routing table itself is not a file: it is + rows in the npm-data volume. + adapters/ + experience.yaml.tmpl templates. setup.py renders these to .yaml, + network.yaml.tmpl filling in the keys it generated. The rendered + provider.yaml.tmpl files hold private keys and are gitignored. + routing-exp.yaml which action goes where. exp sends discover to + routing-network.yaml the network layer and select to the provider; + routing-provider.yaml provider sends publish to the network layer; + network sends discover and publish to discovery + registry/ + schemas/ Participant, ProviderSchema, SchemaRegistry. + Read at startup -- a change needs the registry + service restarted. + imports/ the Keycloak realm + discovery/ + instance.yaml.example optional override; see the compose file + mappings/ + mausamgram/ one file per binding-action: the request and the + agmarknet/ response transformation, in JSONata. These are the + files the adapters fetch over the raw CDN -- the + served copy and the reviewable copy are one file +mock-server/ + mockimd/ the two mock upstreams. Sources only: they are + mockagmarknet/ pulled as published images like everything else. + See mock-server/README.md for the build commands and + for what each deliberately gets wrong. +../postman-collection/ NOT in here -- a sibling of this directory. The + collection plus an environment file, because it is + what you point AT a stack rather than part of one +keys/keys.json generated, gitignored. The private halves of the + three adapter keypairs -- the one file here that + is worth backing up, and the reason setup.py can + be re-run without invalidating what it registered +``` + +## Appendix L — Renaming this directory + +Worth knowing before you pull a rename onto a running host, because Docker will +not warn you. + +**Compose takes its project name from the directory holding the compose file**, +and every named volume is prefixed with it. So this directory becoming +`quick-start` renames all five: docker-deployment_registry-data -> quick-start_registry-data docker-deployment_discovery-data -> quick-start_discovery-data @@ -1206,7 +1477,7 @@ Keycloak shares `registry-data` with the registry, so its realm travels with that one volume -- there is nothing separate to migrate, and equally nothing that survives if you skip it. -## Starting over +## Appendix M — Starting over ```sh docker compose down -v # -v also deletes the registry and discovery data @@ -1216,3 +1487,98 @@ rm -rf keys config/adapters/experience.yaml config/adapters/network.yaml config/ Then start again from `docker compose up -d`. New keys mean new identities, so the provider rows have to be created again too — and the old participant ids cannot be reused. + +## Appendix N — Before you start — the long form + +On the VM: + +- Docker with Compose **v2.24 or newer**, logged in to wherever the images live + if it is private — `docker login ghcr.io`. The version floor is the + `env_file: required: false` on the HyperDX service, which is what lets an + absent `.env.docker` be absent instead of fatal. +- Python 3 and the `cryptography` package — `pip install cryptography` +- 16 GB of RAM if you run the `observability` profile — ClickHouse alone wants + 2-4 GB on top of the two JVM services. 8 GB is workable without it. + +Nothing else. No external API and no tunnel: the two mock upstreams are part +of the stack, so a select has something to answer it the moment it comes up. + +`bin/bootstrap-ubuntu.sh` installs the first two on a fresh Ubuntu VM. + +## Appendix O — Testing it end to end — the long form + +**Quickest path: import `../postman-collection/`.** Six requests, 32 +assertions, nothing to fill in — publish, discover and select for each +capability, with every value already matching this deployment. There are no +registry requests: the registry has no route through the edge, so `setup.py` +seeds it instead. A green run means the stack is healthy +rather than merely answering. + +It sits at the repo root rather than in here, because it is not part of the +compose stack — it is what you point at one, and its environment file exists so +it can be aimed somewhere else. + +The rest of this section is one of those requests as curl, if you would rather +see it than run it. + +```sh +curl -s -X POST http://127.0.0.1:9202/select \ + -H 'Content-Type: application/json' \ + -d '{ + "context": { + "version": "2.0.0", "action": "select", + "networkId": "oan-dev", + "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", + "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", + "timestamp": "2026-09-04T06:12:01.330Z" + }, + "message": { "contract": { "commitments": [ { + "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, + "resources": [ { + "id": "res:mausamgram:point-forecast", + "quantity": 1, + "resourceAttributes": { + "@context": "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld", + "@type": "openagrinet:WeatherObservation", + "subjectCategories": ["Weather"], + "informationMode": "OnDemand", + "supportedObservationTypes": ["Forecast"], + "supportedParameters": ["Rainfall", "Temperature"], + "geographicGranularities": ["Point"], + "location": { "type": "Point", "coordinates": [73.7898, 19.9975] } + } + } ], + "offer": { + "id": "offer:mausamgram:open-data", + "resourceIds": ["res:mausamgram:point-forecast"], + "provider": { "id": "mausamgram-mock", + "descriptor": { "code": "IMD-NWP-01", "name": "IMD Mausamgram NWP" } } + } + } ] } } + }' | python3 -m json.tool +``` + +An `on_select` comes back with one resource per forecast day — three by +default, which is `MOCKIMD_DAYS`. + +The mandi equivalent is the same call to the same endpoint with a `MandiPrice` +resource and `agmarknet-mock` as the provider, and that is the point worth +taking from this section: **one endpoint, two capabilities, and no routing +config in between.** Each provider step builds a binding key out of the +payload it is handed, answers if the key is its own, and passes the payload +through untouched if it is not. Adding a third capability is a plugin and two +registry rows, not a new route. + +Two things about the payload: + +**No party is named, in either direction.** Identity travels in the +`Authorization` header's `keyId`, which names the signer and the key the +registry published for it; a body that declares no caller simply skips the +declared-identity comparison. Nothing needs `bapId` or `bppId`, and the `*Uri` +fields they came with were container-internal addresses that meant nothing +outside this compose network anyway. + +**The experience adapter is the only one that takes an unsigned request.** The +experience app is inside the trust boundary, so there is no network signature +to check — which is what makes this testable with a plain curl. The same call +to the provider adapter on 9200 is rejected unsigned. From 0a96926ce9ca05b5f88356a1491638c5d12c3fab Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 17:37:29 +0530 Subject: [PATCH 75/81] fix: follow the registry plugin rename [OpenAgriNet/network-adapter#4] id: oanregistry -> id: sunbirdRegistry, in all three adapter templates. The adapter branch renamed the package to say what the registry is rather than whose it is, and the .so basename is the plugin id this config refers to, so the two move together or the adapter exits at startup with failed to load Registry plugin (sunbirdRegistry): plugin sunbirdRegistry not found A matching image is published: ghcr.io/ameersohel45/oan-adapter at 642743e, which latest now points at too, so a deployment following latest picks it up on the next explicit pull. Note that pull has to be explicit -- pull_policy is missing and nothing in stack.sh pulls, as .env.example says. Verified end to end rather than by inspection, and it caught a real mistake: rebuilding only ONE of the three adapter images left the other two carrying the old .so, and both died with exactly the error above while the provider came up fine. All three rebuilt, all three started, 51 of 51 on the collection. Worth recording because a partial rebuild fails in a way that looks like a config typo in one adapter rather than a stale image in two. Left alone, as deployment identity rather than code: container_name oan-provider-adapter, OTEL_SERVICE_NAME, and appName/serviceName in the provider template. Renaming those changes what `docker ps` shows and what telemetry reports, which is a separate decision from naming a Go package. --- quick-start/config/adapters/experience.yaml.tmpl | 2 +- quick-start/config/adapters/network.yaml.tmpl | 2 +- quick-start/config/adapters/provider.yaml.tmpl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/quick-start/config/adapters/experience.yaml.tmpl b/quick-start/config/adapters/experience.yaml.tmpl index 35d2ea9..a67b663 100644 --- a/quick-start/config/adapters/experience.yaml.tmpl +++ b/quick-start/config/adapters/experience.yaml.tmpl @@ -69,7 +69,7 @@ modules: plugins: registry: - id: oanregistry + id: sunbirdRegistry config: url: http://registry:8081/api/v1 entity: Participant diff --git a/quick-start/config/adapters/network.yaml.tmpl b/quick-start/config/adapters/network.yaml.tmpl index de2306b..ee2a04d 100644 --- a/quick-start/config/adapters/network.yaml.tmpl +++ b/quick-start/config/adapters/network.yaml.tmpl @@ -72,7 +72,7 @@ modules: plugins: registry: - id: oanregistry + id: sunbirdRegistry config: url: http://registry:8081/api/v1 entity: Participant diff --git a/quick-start/config/adapters/provider.yaml.tmpl b/quick-start/config/adapters/provider.yaml.tmpl index 5fcf52e..ffdb10a 100644 --- a/quick-start/config/adapters/provider.yaml.tmpl +++ b/quick-start/config/adapters/provider.yaml.tmpl @@ -85,7 +85,7 @@ modules: # Serves both halves: the sender's signing key for validateSign, and the # capability call plans the provider steps resolve against. registry: - id: oanregistry + id: sunbirdRegistry config: url: http://registry:8081/api/v1 entity: Participant @@ -238,7 +238,7 @@ modules: plugins: registry: - id: oanregistry + id: sunbirdRegistry config: url: http://registry:8081/api/v1 entity: Participant From 5b4d7898af7dbd18ae6b07f4ba079d29e45147ae Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 17:48:22 +0530 Subject: [PATCH 76/81] refactor: rename routing-exp.yaml, and drop the collection's own README [OpenAgriNet/network-adapter#4] routing-exp.yaml -> routing-experience.yaml, finishing what the experience.yaml.tmpl rename started. Renamed on BOTH sides of the mount, so the container path matches the file rather than preserving the old spelling invisibly, and the template's routingConfig follows it. postman-collection/README.md is removed. Nothing linked to it, and the quick-start README's Step 8 already covered importing the two files, the counts, and why there are no registry requests. Four things in it were not covered anywhere else, so they moved rather than went away: - Point it at another deployment by editing the ENVIRONMENT file, not the collection: Postman resolves an environment variable ahead of a collection variable of the same name, so the loopback defaults survive for the next person. Step 8. - The newman one-liner for a single folder. Step 8. - What the two select requests demonstrate -- same endpoint, same adapter, different domain packages answering, because dispatch is a binding key built from the payload and not a route. Appendix O. - Why networkAdapterUrl is a variable no request uses. It is the other adapter a deployment exposes publicly, and its endpoints verify signatures, which Postman does not do -- so the variable exists to give the address a home, not because a request is missing. Without this written down it reads like an oversight and invites someone to "fix" it. Appendix O. --- postman-collection/README.md | 58 ------------------- quick-start/README.md | 38 ++++++++++-- .../config/adapters/experience.yaml.tmpl | 2 +- ...uting-exp.yaml => routing-experience.yaml} | 0 quick-start/docker-compose.yml | 2 +- 5 files changed, 36 insertions(+), 64 deletions(-) delete mode 100644 postman-collection/README.md rename quick-start/config/adapters/{routing-exp.yaml => routing-experience.yaml} (100%) diff --git a/postman-collection/README.md b/postman-collection/README.md deleted file mode 100644 index 740bb16..0000000 --- a/postman-collection/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Postman collection - -Two files. Import both. - - api-collection.json the requests - local_postman_environment.json where your deployment's URLs go - -**The collection alone works against a tunnel.** Every URL variable defaults to -loopback, because the stack publishes its ports on the VM's loopback only: - - ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ - -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm - -**The environment is how you point it somewhere else.** Import it, select it in -the environment dropdown, and edit the six URLs at the top — the experience, -network and provider adapters, the registry, Keycloak and discovery. Postman -resolves an environment variable ahead of a collection variable of the same -name, so nothing in the collection needs touching and the loopback defaults -stay intact for the next person. - -No VM hostname or address is committed in either file. Deployment addresses are -shared separately, and the environment file is the place to put them. - -## Two folders, one per capability - - 1. WeatherObservation 1. Publish 2. Discover 3. Select - 2. MandiPrice 1. Publish 2. Discover 3. Select - -Six requests, 32 assertions. Run a folder top to bottom the first time -- -Publish seeds the catalogue Discover looks for -- and after that any request -works on its own: - - newman run api-collection.json --folder "2. MandiPrice" - -The two Select requests are the pair worth comparing. They hit the same -endpoint on the same adapter and different domain packages answer them, because -each provider step recognises its own binding key from the payload and passes -through anything else. Nothing routes by URL, path or domain. - -## No registry requests here - -Deliberate. The registry has no route through the gateway and publishes on -loopback only, so nothing in a shared collection could reach it. -`bin/setup.py` seeds all of it -- five participants and both capability -bindings -- from the same `.env` the adapter configs are rendered from, which is -what keeps the two from disagreeing. - -To look at a registry row, tunnel to the VM and use `Participant/search` -directly; the quick-start README has the curl. - -## networkAdapterUrl - -Present as a variable, used by no request. Discover reaches the network adapter -through the experience adapter and publish through the provider adapter, so -nothing here calls it directly. It is there because it is the other adapter a -deployment exposes publicly: its `/publish` and `/discover` both verify -signatures, so a network peer calls it directly. Signing is not something -Postman does, so those calls are not scripted. diff --git a/quick-start/README.md b/quick-start/README.md index de696d5..474012f 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -203,7 +203,7 @@ docker compose logs network-adapter | grep 'Server listening' The consumer's edge. Sends `discover` to the network layer and `select` straight to the provider layer — which action goes where is -`config/adapters/routing-exp.yaml`, not a code path. +`config/adapters/routing-experience.yaml`, not a code path. **Config it needs** @@ -246,8 +246,22 @@ against it — so run publish before discover the first time. A green run means the registry is seeded, both adapters sign and verify, both mappings work, and discovery is indexing. -There are no registry requests in the collection deliberately — the registry -has no route through the edge, so `setup.py` seeds it instead. +Or without Postman: + +```sh +newman run ../postman-collection/api-collection.json --folder "2. MandiPrice" +``` + +**To point it at another deployment, edit the environment file, not the +collection.** Postman resolves an environment variable ahead of a collection +variable of the same name, so the loopback defaults stay intact for the next +person. No deployment address is committed in either file, deliberately. + +There are no registry requests in the collection either — the registry has no +route through the edge, so `setup.py` seeds it instead. + +→ Appendix O for what the two `select` requests demonstrate, and why one +variable is deliberately called by nothing. --- @@ -1399,7 +1413,7 @@ config/ experience.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered provider.yaml.tmpl files hold private keys and are gitignored. - routing-exp.yaml which action goes where. exp sends discover to + routing-experience.yaml which action goes where. it sends discover to routing-network.yaml the network layer and select to the provider; routing-provider.yaml provider sends publish to the network layer; network sends discover and publish to discovery @@ -1514,6 +1528,22 @@ registry requests: the registry has no route through the edge, so `setup.py` seeds it instead. A green run means the stack is healthy rather than merely answering. +**The two `select` requests are the pair worth comparing.** They hit the same +endpoint on the same adapter, and different domain packages answer them — +because each provider step builds a binding key from the payload, serves the +request if the key is its own, and passes through anything else. Nothing routes +by URL, path or domain. That is the whole dispatch mechanism, and these two +requests are what demonstrate it. + +**`networkAdapterUrl` is a variable no request uses, on purpose.** `discover` +reaches the network adapter through the experience adapter and `publish` +through the provider adapter, so nothing in the collection calls it directly. +It is listed because it is the other adapter a deployment exposes publicly: +its `/publish` and `/discover` both verify signatures, so a network peer calls +it directly. Signing is not something Postman does, so those calls are not +scripted — the variable is there so the address has somewhere to live, not +because a request is missing. + It sits at the repo root rather than in here, because it is not part of the compose stack — it is what you point at one, and its environment file exists so it can be aimed somewhere else. diff --git a/quick-start/config/adapters/experience.yaml.tmpl b/quick-start/config/adapters/experience.yaml.tmpl index a67b663..196dfd4 100644 --- a/quick-start/config/adapters/experience.yaml.tmpl +++ b/quick-start/config/adapters/experience.yaml.tmpl @@ -112,7 +112,7 @@ modules: router: id: router config: - routingConfig: /app/config/routing-exp.yaml + routingConfig: /app/config/routing-experience.yaml steps: - validateSchema diff --git a/quick-start/config/adapters/routing-exp.yaml b/quick-start/config/adapters/routing-experience.yaml similarity index 100% rename from quick-start/config/adapters/routing-exp.yaml rename to quick-start/config/adapters/routing-experience.yaml diff --git a/quick-start/docker-compose.yml b/quick-start/docker-compose.yml index a472b78..2b73bbc 100644 --- a/quick-start/docker-compose.yml +++ b/quick-start/docker-compose.yml @@ -452,7 +452,7 @@ services: OTEL_SERVICE_NAME: oan-exp-adapter volumes: - ./config/adapters/experience.yaml:/app/config/adapter.yaml:ro - - ./config/adapters/routing-exp.yaml:/app/config/routing-exp.yaml:ro + - ./config/adapters/routing-experience.yaml:/app/config/routing-experience.yaml:ro ports: - "127.0.0.1:${EXP_ADAPTER_PORT}:9202" From 25ebdc6ed151a9bde77898017dbd4dcd24daee91 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 17:53:44 +0530 Subject: [PATCH 77/81] docs: condense the appendices, and redraw the flow diagram [OpenAgriNet/network-adapter#4] The appendices were the original sections moved verbatim, which fixed the ordering problem and left the length one: 1224 lines of reference behind a 375-line walkthrough. Now 561. The cuts are prose, not facts. Every command, table, gotcha and reason is still there -- the certificate traps, the /publish deny and why it is a mounted file rather than a click, the Keycloak X-Forwarded-* trap, the append-only registry, the bind-mount-directory failure, the if/then/else the validator never evaluates, the npm-letsencrypt duplicate-certificate limit. What went is the paragraph of build-up in front of each one. Two sections went entirely. "Before you start -- the long form" duplicated Steps 1 and 2 with nothing added. "Testing it end to end -- the long form" duplicated Step 8 except for two explanations, which are now Appendix N on their own: what the two select requests demonstrate, and why networkAdapterUrl is a variable no request uses. THE DIAGRAM WAS WRONG, not just verbose. It drew a box around the experience adapter and put the network and provider adapters INSIDE it, with an arrow leaving the closed box afterwards -- so it read as one service containing two others. Redrawn as what it is: the consumer reaches the experience adapter, which fans out to the network adapter for discover and the provider adapter for select, and the provider calls the mocks. publish running the other way and the shared registry reads are now stated beside it rather than crammed into the drawing. Also dropped a line from Appendix K listing bin/fetch-schemas.sh, which was removed several commits ago. Listing a file that is not there is worse than not listing it. Verified: 4 tables well-formed, code fences balanced, every appendix reference in the body resolves, and no reference left pointing at a dropped section. --- quick-start/README.md | 1381 +++++++++++------------------------------ 1 file changed, 357 insertions(+), 1024 deletions(-) diff --git a/quick-start/README.md b/quick-start/README.md index 474012f..4a7aad9 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -9,29 +9,28 @@ minutes, most of it waiting for Keycloak. ## The stack in one picture ``` - consumer - │ - ▼ - ┌──────────── experience adapter ────────────┐ the caller's edge - │ │ - │ discover ──► network adapter ──► discovery service - │ │ - │ select ────────────────────────► provider adapter - └────────────────────────────────────────────┘ │ - ▼ - mockimd · mockagmarknet - - every adapter reads the registry: who signed this, and where does - this capability's provider live + consumer + │ + ▼ + experience adapter the only one a consumer calls + │ + ├── discover ──► network adapter ──► discovery service + │ + └── select ──► provider adapter ──► mockimd + └─► mockagmarknet ``` -A `discover` asks the network layer what exists. A `select` goes straight to -the provider layer, which calls the upstream and answers in the same HTTP -round trip — there is no callback. +`publish` runs the other way: the provider adapter sends it to the network +adapter, which files the catalogue in discovery. -That is the request order. The **startup** order is the reverse: the -experience adapter depends on the other two, so Compose brings them up first. -The steps below are in startup order, so they work top to bottom. +All three adapters read the registry — who signed this request, and where this +capability's provider lives. + +A `select` answers in the same HTTP round trip — there is no callback. + +That is the request order. **Startup order is the reverse**: the experience +adapter depends on the other two, so Compose brings them up first. The steps +below follow startup order, so they work top to bottom. ## Which path are you on? @@ -260,7 +259,7 @@ person. No deployment address is committed in either file, deliberately. There are no registry requests in the collection either — the registry has no route through the edge, so `setup.py` seeds it instead. -→ Appendix O for what the two `select` requests demonstrate, and why one +→ Appendix N for what the two `select` requests demonstrate, and why one variable is deliberately called by nothing. --- @@ -384,74 +383,46 @@ Reference, read on demand. Nothing here is a step. | **K** | The layout | | **L** | Renaming this directory | | **M** | Starting over | -| **N** | Before you start, and bringing it up — the long form | -| **O** | Testing it end to end — the long form | - ---- +| **N** | What the collection demonstrates | ## Appendix A — What is here, and what is not -Running here: - -- **registry** — SunbirdRC, plus its Postgres and Keycloak. Holds who is on the - network, their public keys, and which upstream API answers which capability. - Published on the VM's loopback only, and deliberately given no route through - the gateway — see below. -- **discovery** — catalogue search, plus its own Postgres. -- **three adapters** — experience, network and provider. Same image, three - configs. -- **two mock upstreams** — one standing in for Mausamgram's forecast API, one - for Agmarknet's Vistaar prices. Sources in `mock-server/`; they are pulled as - published images like everything else. They exist so the stack answers a - select end to end out of the box, with no external API and no ngrok tunnel. - Loopback only, and the adapter reaches them by compose service name rather - than through the published port. -- **gateway** — Nginx Proxy Manager, the only container that publishes on a - routable interface. Routes to the three adapters, and issues and renews the - Let's Encrypt certificates from its own UI. Profile `reverse-proxy`. -- **hyperdx** — ClickStack: OTLP ingest, ClickHouse, and the UI over it. - Profile `observability`. - -The last two are behind profiles because neither is needed to exercise the -stack, and HyperDX is the heaviest thing here. - -Deliberately **not** here: - -- **a route to the registry.** It is reachable from inside the compose network - and over an SSH tunnel to the VM, and from nowhere else. Nothing in front of - it authenticates, and SunbirdRC uses POST for both reads and writes, so a - route would expose creates as readily as searches. This is why `bin/setup.py` - seeds everything: with no public registry there is no second way to write a - row, and a Postman request could not do it. -- **a real provider API.** The mocks answer the same shapes. Pointing a - capability at something real is a registry write — a new Participant and - ProviderSchema row, made from inside the stack — and a base URL in `.env`. - The provider adapter never holds that address in a config file; it reads it - from the registry per request. +Running: **registry** (SunbirdRC + Postgres + Keycloak), **discovery** +(catalogue search + Postgres), **three adapters** (same image, three configs), +**two mock upstreams** standing in for Mausamgram and Agmarknet. Behind +profiles: **nginx-proxy-manager** (`reverse-proxy`, the only container on a +routable interface) and **hyperdx** (`observability`, ClickStack — the +heaviest thing here). -## Appendix B — Reaching it +Deliberately absent: -Two doors, and which one you use depends on what you are reaching. +- **A route to the registry.** Reachable from inside the Compose network and + over an SSH tunnel, nowhere else. Nothing in front of it authenticates, and + SunbirdRC uses POST for reads *and* writes, so a route would expose creates + as readily as searches. That is why `setup.py` seeds everything — there is no + second way in. +- **A real provider API.** The mocks answer the same shapes. Pointing a + capability at something real is a registry write plus a base URL in `.env`; + the adapter reads the address per request rather than holding it. -### The adapters — through Nginx Proxy Manager +## Appendix B — Reaching it + +### The adapters, through Nginx Proxy Manager -NPM owns 80 and 443 and is the whole public surface. Unlike a config file, its -routing table is **rows in a SQLite database** inside the `npm-data` volume — -so the setup below is a one-time click-through, and that volume is the only -copy of the result. Back it up. +NPM owns 80 and 443 and is the whole public surface. Its routing table is +**rows in a SQLite database** in the `npm-data` volume, not a config file — so +setup is a one-time click-through and that volume is the only copy. Back it up. -**First boot.** Tunnel to the admin UI (it is bound to loopback on purpose, -see below) and change the shipped login immediately: +**First boot.** Tunnel to the admin UI and change the shipped login before +creating anything: ```sh -ssh -L 81:127.0.0.1:81 -N you@the-vm +ssh -L 81:127.0.0.1:81 -N you@the-vm # then http://127.0.0.1:81 ``` -Open `http://127.0.0.1:81`. It logs in with `admin@example.com` / `changeme`, -which is live from first boot until you change it, and it forces a change on -first use. Do that before creating anything. +It logs in with `admin@example.com` / `changeme`, live from first boot. -**Then one proxy host per adapter.** Hosts → Proxy Hosts → Add Proxy Host: +**One proxy host per adapter.** Hosts → Proxy Hosts → Add: | Domain | Forward Hostname | Port | Then | |---|---|---|---| @@ -459,407 +430,165 @@ first use. Do that before creating anything. | `network.oan.example.com` | `network-adapter` | 9201 | — | | `provider.oan.example.com` | `provider-adapter` | 9200 | — | -Scheme `http` for all three: TLS terminates at NPM, and the hop to an adapter -is inside `oan-edge`. Turn on **Block Common Exploits**; leave **Websockets -Support** off, since nothing here uses them. +Scheme `http` for all three — TLS terminates at NPM and the hop inward is +inside `oan-edge`. Turn on **Block Common Exploits**; leave Websockets off. -Three hosts rather than one host with path prefixes, because a rate limit or a -block then attaches to a whole hostname instead of being expressed as a regex -in a textarea — and each gets its own certificate. A request arriving with a -`Host` NPM does not know gets NPM's default page, not an adapter. +Three hosts rather than one with path prefixes, so a rate limit or a block +attaches to a hostname instead of a regex in a textarea, and each gets its own +certificate. An unknown `Host` gets NPM's default page, not an adapter. -**Certificates.** SSL tab → Request a new SSL Certificate → Force SSL → HTTP -Validation. That needs two things to be true, and both are easy to miss: +**Certificates.** SSL tab → Request a new certificate → Force SSL → HTTP +validation. Two things must be true, and both are easy to miss: -- a public DNS **A record** per hostname, pointing at the VM's address — an - Elastic IP, unless you enjoy redoing this after every stop/start; +- a public DNS **A record** per hostname, pointing at the VM — use a static + address unless you enjoy redoing this after every stop/start; - **port 80 open to `0.0.0.0/0`**, not to your address. Let's Encrypt fetches - `http:///.well-known/acme-challenge/…` from its own servers, whose - addresses you do not get to enumerate. A security group scoped to your IP - makes issuance fail with a challenge timeout, which looks nothing like a - firewall problem in the NPM log. - -If 80 must stay closed, use **DNS Validation** instead: NPM ships the certbot -Route 53 plugin, so you give it an access key with `route53:ChangeResourceRecordSets` -on the zone and it never needs an inbound request. That is the better answer on -AWS anyway, and it is the only one that works for a wildcard. - -Renewal is NPM's job from then on, and it uses the same validation method — so -a DNS record or an SG rule that was only temporarily correct will fail silently + the challenge from its own servers, whose addresses you cannot enumerate. A + security group scoped to your IP fails with a challenge timeout that looks + nothing like a firewall problem. + +If 80 must stay closed, use **DNS validation** — NPM ships the Route 53 plugin, +so an access key with `route53:ChangeResourceRecordSets` needs no inbound +request. It is also the only option for a wildcard. Renewal reuses whichever +method you chose, so a temporarily-correct DNS record or SG rule fails silently in sixty days. -### What is *not* reachable, and why that holds +### `POST /publish` returns 403 on all three hosts -`POST /publish` returns 403 on all three hosts. This is not optional -hardening — it is the one thing standing between the public internet and an -unauthenticated write into the catalogue. +Not optional hardening. The provider adapter's module at `/` verifies the +sender's signature; `oanProviderPublish`, on the exact path `/publish`, has +**no signature check at all**, because its intended caller is the provider's +own catalogue system inside the trust boundary. A proxy host pointed at +`provider-adapter:9200` therefore exposes `/publish` to anyone. -The provider adapter mounts two modules. The one at `/` verifies the sender's -signature against the registry; `oanProviderPublish`, on the exact path -`/publish`, has **no signature check at all**, because its intended caller is -the provider's own catalogue system inside the trust boundary. A proxy host pointed at -`provider-adapter:9200` therefore exposes `/publish` to anyone. NPM's UI -offers no way to route a host while withholding one path, so the block lives in -`config/reverse-proxy/npm-custom/server_proxy.conf`, which NPM includes in **every** -proxy host's server block automatically — a mounted file, not a click, and so -not something to remember on one host out of three. +NPM's UI cannot route a host while withholding one path, so the block lives in +`config/reverse-proxy/npm-custom/server_proxy.conf`, which NPM includes in +**every** proxy host's server block — a mounted file, not a click, so not +something to remember on one host out of three. To let a real catalogue system publish, give it a tunnel or put it in the VPC and let it reach `provider-adapter:9200` directly. Do not turn that `deny` into -an `allow`: an endpoint with no credential to check does not belong on a public -edge, and "an address allowlist in front of it" is a statement about the -network, which is where it should be made. - -And the deeper reason a UI-configured proxy is safe here at all: **NPM sits on -`oan-edge` only.** "Forward Hostname" is a free-text field, so anyone with the -admin password can type `registry`, `keycloak` or `discovery-db` into it — and -on that network none of those names resolve and none of those addresses are -routable. The blast radius of a wrong click is bounded to the tier that is -public anyway. Moving NPM onto `oan-internal` to "make things easier" would -remove that bound and put the registry's write API one form field away from the -internet. +an `allow`. -### Which nginx config is loaded, and which is a paste job - -Worth being exact about, because the two look alike in the repo: +### Which nginx config loads itself, and which is a paste job | File | How it applies | |---|---| -| `config/reverse-proxy/npm-custom/http_top.conf` | **Automatic.** NPM includes it at the top of its `http` block. Declares the `exp` rate-limit zone and `limit_req_status 429`. | -| `config/reverse-proxy/npm-custom/server_proxy.conf` | **Automatic.** Included in every proxy host's server block. Holds the `/publish` deny. | -| `config/reverse-proxy/npm-advanced/exp.conf` | **Manual.** Paste into the experience host's Advanced tab. Applies `limit_req` to that host only, since a 10 r/s ceiling on signed peer traffic would throttle for no security gain. | +| `npm-custom/http_top.conf` | **Automatic**, top of the `http` block. The `exp` rate-limit zone and `limit_req_status 429`. | +| `npm-custom/server_proxy.conf` | **Automatic**, every server block. The `/publish` deny. | +| `npm-advanced/exp.conf` | **Manual** — paste into the experience host's Advanced tab. `limit_req` for that host only; a 10 r/s ceiling on signed peer traffic would throttle for no gain. | The manual one is in a file anyway because NPM's Advanced field is a textarea -in a database row: nothing diffs it and nothing reviews it. Keeping the source -here means the rule can be read even though the running copy cannot. +in a database row — nothing diffs it and nothing reviews it. ### Adding a route for another service -Deliberately two steps, and the first one is in git rather than in the UI. - -NPM is on `oan-edge`, where only the three adapters resolve. A proxy host -pointed at `registry` or `hyperdx` does not quietly work — it 502s, because -there is no route. So publishing something new is a change to -`docker-compose.yml` that a reviewer sees, followed by a click. The UI alone -cannot widen what is public. That is the property worth keeping; everything -below is about how to spend it deliberately. - -**Step 1 — put the service on `oan-edge`.** In `docker-compose.yml`, add the -network to that service. Keep `oan-internal` if it talks to anything else in -the stack, and keep the loopback publish or drop it as you like — NPM reaches -the container port directly, not the published one: - -```yaml - some-service: - networks: [oan-internal, oan-edge] -``` - -Then `docker compose up -d some-service nginx-proxy-manager`. NPM needs the -restart to pick up a name it could not resolve before. - -**Step 2 — add the proxy host.** UI → Hosts → Proxy Hosts → Add Proxy Host. -Domain `some.oan.example.com`, scheme `http`, Forward Hostname the **compose -service name** (`some-service`, not `oan-some-service` and not an IP), Forward -Port the **container** port. Then the SSL tab as with the adapters, and a DNS -A record before you request the certificate. - -**A service that is not in this compose file** — a provider API on another -host, something in the VPC — needs no step 1 at all. NPM has egress, so put -its address or hostname straight into Forward Hostname. Nothing about the -network split is involved, and nothing about that service becomes reachable -from inside this stack. - -**A second path on an existing domain** does not need a new host either. Open -the host → Custom Locations → add e.g. `/v2` forwarding to another service. -That keeps one certificate and one DNS record, at the cost of NPM's generated -config growing a location block you cannot see in the UI's main view. - -#### Worked example: discovery - -`discovery` is on `oan-internal` only, so this is the two-step case — the one -where the network split does the work. - -**Step 1, attach it to `oan-edge`** in `docker-compose.yml`: +Two steps, and the first is in git rather than the UI. NPM sits on `oan-edge`, +where only the three adapters resolve, so a proxy host pointed at `registry` or +`hyperdx` 502s rather than quietly working. **The UI alone cannot widen what is +public** — that is the property worth keeping. -```yaml - discovery: - networks: [oan-internal, oan-edge] -``` - -then `docker compose up -d discovery`. Until this, NPM cannot resolve the name -`discovery` at all and a host pointed at it fails DNS rather than working. - -**Step 2, create the host.** Hosts → Proxy Hosts → Add: - -| Field | Value | -|---|---| -| Domain | `discovery.oan.example.com` | -| Scheme | `http` | -| Forward Hostname | `discovery` — the compose service name, not `oan-discovery` | -| Forward Port | `8080` — the **container** port. Not `DISCOVERY_PORT`, which is only what loopback publishes it as | -| Block Common Exploits | on | +1. **Put the service on `oan-edge`** in `docker-compose.yml` + (`networks: [oan-internal, oan-edge]`), then + `docker compose up -d some-service nginx-proxy-manager`. NPM needs the + restart to resolve a name it could not see before. +2. **Add the proxy host.** Forward Hostname is the **Compose service name** + (`discovery`, not `oan-discovery`, not an IP); Forward Port is the + **container** port, not what loopback publishes it as. Then SSL, and a DNS + record before requesting the certificate. -**Step 3, put an Access List on it,** because discovery answers -unauthenticated and `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` in this -build. Nothing behind the edge will refuse a caller, so the edge is the only -authentication there is. +A service **not** in this Compose file needs no step 1 — NPM has egress, so put +its address straight into Forward Hostname. A **second path on an existing +domain** needs no new host either: Custom Locations, one certificate, one DNS +record. -**Check it:** +Anything answering unauthenticated needs an **Access List** on top. Discovery +does — `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` in this build, so the +edge is the only authentication there is. Check both directions, because the +failure is silent: ```sh -curl -s -o /dev/null -w '%{http_code}\n' \ - https://discovery.oan.example.com/health -u user:pass # 200 - -curl -s -o /dev/null -w '%{http_code}\n' \ - https://discovery.oan.example.com/health # 401 +curl -s -o /dev/null -w '%{http_code}\n' https://discovery.oan.example.com/health -u user:pass # 200 +curl -s -o /dev/null -w '%{http_code}\n' https://discovery.oan.example.com/health # 401 ``` -If the second returns 200 the Access List is not attached, and that failure is -silent — worth re-running after any NPM change. - -#### The registry is the one you do not route - -It will look like the obvious candidate: `POST /api/v1/Participant/search` -takes no token, and it is exactly the call a network peer needs. Route it -anyway and you have published more than that. - -SunbirdRC uses POST for **both** reads and writes — `/Participant/search` -reads, `/Participant` creates — so no method rule tells one from the other. A -proxy host forwards the whole API. What keeps writes out today is not the -route: it is that nothing outside the VM can mint a Keycloak token, because -Keycloak publishes on `127.0.0.1`. That is a decision made elsewhere in the -compose file, and a registry route would depend on it silently. Publish -Keycloak later for an unrelated reason and the registry's write surface opens -with it, with nothing in the route changing to say so. - -So `registry` is on `oan-internal` only and stays there. NPM cannot resolve -the name, which means the refusal is structural rather than a proxy host -somebody remembered not to create. - -Two consequences worth knowing, because both look like bugs otherwise: - -- **`bin/setup.py` has to seed everything** — all five participants and both - capability bindings — since there is no other way to write a row. It runs on - the VM against `127.0.0.1`. -- **the Postman collection has no registry request.** Not an omission; one - could not work. - -Reaching it to look at a row is an SSH tunnel, covered further down. - -If a peer genuinely needs to read participants from outside, the answer is a -route to something that serves only that read — not a route to the registry. - -#### Before you route the ones already here - -Several internal services will look like obvious candidates. They are not -equivalent: - -| | What routing it publishes | -|---|---| -| **discovery** | Read-mostly catalogue search. The most defensible of these, and still: it answers unauthenticated, and `AUTH_ENABLE_SIGNATURE_VERIFICATION` is `false` with nothing behind it in this build. Put an Access List on it. | -| **registry** | No — see above. A proxy host forwards reads and writes alike, and it is off `oan-edge` so one cannot be created. | -| **keycloak** | An admin console with a realm imported from a file that ships `no-user` / `no-user-password` and an admin-api client secret. Do not publish it. | -| **hyperdx** | `clickstack-local` runs single-user with **no login at all**. Publishing it hands over every trace and log the stack has collected. If it must be shared, switch to `clickstack-all-in-one` and set up a team first. | -| **the two mocks** | Pointless and confusing: they exist to be called from inside by the provider adapter, and they invent their data. Nothing outside has a reason to reach them. | -| **registry-db, discovery-db** | No. Use `docker compose exec`, or a tunnel. | - -The pattern: publishing a service that has no authentication of its own means -the edge is now its only authentication. NPM can be that, but only if you say -so explicitly. - -#### Putting authentication in front of one +### The registry is the one you do not route -NPM's **Access Lists** are the built-in answer, and they are per-host: UI → -Access Lists → Add. Two independent tabs — - -- **Authorization**: username/password pairs, enforced as HTTP basic auth. -- **Access**: `allow`/`deny` rules by address or CIDR. - -**Satisfy Any** decides how they combine, and the default is the one people -get wrong. *Any* means an allowed address gets in without a password — fine -for "the office network, or a password from anywhere". Turn it **off** for -"an allowed address **and** a password", which is what you want in front of -anything that has no auth of its own. - -Then assign the list on the proxy host's Details tab. It applies to the whole -host, including any Custom Locations under it. - -Basic auth is not a substitute for a real access control, and it travels in a -header on every request — so it is worth having only over HTTPS, which is the -other reason to get the certificate before the route. - -#### The cost of each addition - -Every service you attach to `oan-edge` is one form field away from being -public, because that is exactly what the network split buys and spending it is -irreversible by clicking. Keeping `oan-edge` small is what keeps "someone got -into the NPM admin UI" a bounded incident rather than an open question about -the registry. - -So: add the network in the same change that adds the proxy host, not in -advance "so it's ready". And when a route is retired, take the service back -off `oan-edge` rather than only deleting the host in NPM. - -### Everything else — through an SSH tunnel - -The registry, Keycloak, discovery, the HyperDX UI and NPM's own admin UI -publish on `127.0.0.1` only: - -```sh -ssh -L 81:127.0.0.1:81 \ - -L 8080:127.0.0.1:8080 \ - -L 8081:127.0.0.1:8081 \ - -L 8082:127.0.0.1:8082 \ - -L 8085:127.0.0.1:8085 \ - -N you@the-vm -``` - -NPM admin on 81, Keycloak on 8080, the registry on 8081, discovery on 8082, -HyperDX on 8085 (adjust to your `.env`). Both Postgres instances publish no -port at all; reach them with `docker compose exec registry-db psql …`. - -The admin UI is on loopback rather than published-and-firewalled, which is what -the port comment in most NPM compose examples suggests. It ships with a known -default login and it is the one surface on this box that can mint certificates -and re-point every public route; a security group is a second system to keep in -step with that, and a loopback bind is not. - -There is no `BIND_ADDR` any more. It used to move every published port onto the -public interface at once, which is a footgun once something exists to expose -the one tier that should be reachable. +It looks like the obvious candidate — `POST /api/v1/Participant/search` takes +no token and is exactly what a peer needs. But SunbirdRC uses POST for reads +*and* writes, so no method rule tells them apart and a proxy host forwards the +whole API. What keeps writes out today is not the route: it is that nothing +outside the VM can mint a Keycloak token, because Keycloak publishes on +loopback. A registry route would depend on that silently. ## Appendix C — Startup order, and why it is that order -```sh -cp .env.example .env -make up -``` - -Read `.env` first. Two things in it matter before a first run: - -- **the credentials.** All shipped defaults, and this file is public. Change - them. -- `ADAPTER_IMAGE`, `DISCOVERY_IMAGE`, `MOCKIMD_IMAGE`, `MOCKAGMARKNET_IMAGE` — - the tags published for this environment. `TAG` pins discovery on its own: - `TAG=v0.3.1 make up` deploys a known build instead of whatever `latest` - points at today. Nothing is built here; everything is pulled. - -The rest has working defaults and is commented where the reasoning is not -obvious. - -`make up` runs five steps in the order they have to happen. `make up-core` -stops after step 3, which is enough to exercise the stack: - ``` 1. registry and discovery (also registry-db, keycloak, discovery-db) -2. bin/setup.py keys, five registry participants, adapter configs +2. bin/setup.py keys, five participants, two bindings, three configs 3. mock upstreams, then the three adapters -4. nginx-proxy-manager the public edge -- 80 and 443, all interfaces +4. nginx-proxy-manager the public edge — 80 and 443, all interfaces 5. hyperdx ClickStack ``` -Step 2 is the one to understand. It generates a keypair per adapter into -`keys/keys.json`, writes five participants and two capability bindings into -the registry, and renders the three adapter configs from the templates in -`config/adapters/`. **Nothing has to be created by hand afterwards** — and -nothing can be, from outside the VM, because the registry has no route. +`make up` runs all five; `make up-core` stops after 3, which is enough to +exercise the stack. -Step order is not cosmetic. An adapter config is a bind-mounted *file*, and -Docker creates a *directory* at any bind-mount source that is missing — so an -adapter started before step 2 wedges on `adapter.yaml: is a directory` and -leaves a directory where step 2 needs a file. This is the entire reason the -Makefile exists rather than a line in the README saying "run these in order". -`make up` gets it right; a bare `docker compose up -d` on a fresh checkout -does not. `bin/setup.py` refuses with an explanation if it finds one of those -directories — delete them and re-run. +**The order is not cosmetic.** An adapter config is a bind-mounted *file*, and +Docker creates a *directory* at any missing bind-mount source — so an adapter +started before step 2 wedges on `adapter.yaml: is a directory` and leaves a +directory where step 2 needs a file. This is why the Makefile exists rather +than a README line saying "run these in order". `setup.py` refuses with an +explanation if it finds one; delete them and re-run. -Re-running `make up` is safe. `setup.py` reuses the keys in `keys/keys.json` -and skips registry rows that already exist, so it converges rather than -failing on the second run. +Re-running is safe: `setup.py` reuses `keys/keys.json` and skips rows that +already exist. -Check it: +Verify: ```sh make ps curl -s -X POST http://127.0.0.1:8081/api/v1/Participant/search \ - -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool -``` + -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool # five -Five participants: three adapters and two upstreams. That is what `setup.py` -seeded, and that curl only works on the VM itself or through a tunnel. - -And through the gateway, once the proxy hosts exist: - -```sh -# the routed surface -curl -s -o /dev/null -w '%{http_code}\n' \ - https://exp.oan.example.com/search # reaches the adapter - -# the two that matter more -curl -s -o /dev/null -w '%{http_code}\n' \ - https://provider.oan.example.com/publish # 403 -- the deny is loaded -curl -s -o /dev/null -w '%{http_code}\n' \ - http://the-vm-ip/ # NPM default page, no adapter +curl -s -o /dev/null -w '%{http_code}\n' https://provider.oan.example.com/publish # 403 ``` -That 403 is the check worth repeating after any NPM change: it is the only -evidence that `npm-custom/server_proxy.conf` is still mounted, and losing the -mount silently opens an unauthenticated catalogue write. +That 403 is the check worth repeating after **any** NPM change — it is the only +evidence `server_proxy.conf` is still mounted. ## Appendix D — What is in the registry, and why you did not create it -`bin/setup.py` wrote all of it. Nothing in this section is a step to perform — -it is what to look at when something does not match. - -**Three `node` rows, one per adapter.** These are network identities: an id, a -role, and the public halves of a keypair. The private halves stay in -`keys/keys.json` on the VM and are never in the registry. A signature between -adapters is verified against these rows. - -Roles are `consumer`, `provider` and `network`, and they apply to `node` rows -only. A node needs at least one key, published as bare base64 with no encoding -label in front of it. +`setup.py` wrote all of it. Nothing here is a step; it is what to look at when +something does not match. -**Two `upstream` rows, one per mock API.** An upstream is an ordinary HTTP API -this deployment calls. It signs nothing and nothing verifies it, so it needs no -role and no keys. It holds a `baseUrl` — here a compose service name, because -these are reached from inside the network and nowhere else. +**Three `node` rows, one per adapter** — an id, a role (`consumer`, `provider`, +`network`) and the public halves of a keypair. Private halves stay in +`keys/keys.json` and are never in the registry. Keys are published as bare +base64, no encoding label. -No credential for an upstream lives in the registry either. The adapter -presents credentials from its own config, which names *environment variables* -rather than values: the mandi binding uses `queryValueEnv`, and `MANDI_TOKEN` -reaches the container as an env var. +**Two `upstream` rows, one per mock** — an ordinary HTTP API this deployment +calls. It signs nothing, so it needs no role and no keys. Holds a `baseUrl`, +here a Compose service name. No upstream credential lives in the registry +either: the adapter config names *environment variables*, not values. -**Two `ProviderSchema` rows, one per capability.** This is the row that says -which upstream answers which capability and how to call it — method, path, -timeout, retries, and the URL of the mapping file. Its `bindingKey` is -`participantId|capabilityCode`: +**Two `ProviderSchema` rows, one per capability** — which upstream answers +which capability and how to call it: method, path, timeout, retries, and the +mapping URL. Its `bindingKey` is `participantId|capabilityCode`: ``` mausamgram-mock|openagrinet:WeatherObservation agmarknet-mock|openagrinet:MandiPrice ``` -Those two strings are the hinge of the whole thing. The provider adapter builds -the same key out of each incoming payload — the provider id and the capability -`@type` it carries — and a step answers only when the key it was configured -with matches. `setup.py` renders those keys into `config/adapters/provider.yaml` -from the same `.env` values it seeds the registry from, which is what stops the -two from drifting. +Those two strings are the hinge. The provider adapter builds the same key from +each payload — the provider id and the capability `@type` it carries — and a +step answers only when the key matches its own. `setup.py` renders those keys +into `provider.yaml` from the same `.env` it seeds the registry from, which is +what stops the two drifting. -### Looking at it - -Only from the VM, or through a tunnel: - -```sh -curl -s -X POST http://127.0.0.1:8081/api/v1/Participant/search \ - -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool - -curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema/search \ - -H 'Content-Type: application/json' -d '{"filters":{}}' | python3 -m json.tool -``` - -Search takes no token. Writes do, and the token request has a trap in it: +**Looking at it** — from the VM or a tunnel. Search takes no token; writes do, +and the token request has a trap: ```sh TOKEN=$(curl -s -X POST \ @@ -870,127 +599,41 @@ TOKEN=$(curl -s -X POST \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') ``` -Those two `X-Forwarded-*` headers are not optional, and `keycloak:8080` is the -**container-internal** address on purpose — not whatever `KEYCLOAK_PORT` -publishes it as. Keycloak builds the token's issuer from these headers and the -registry validates that issuer against the internal address. Get it wrong and -the registry rejects the token with a 401 and an empty body. - -### Giving an upstream its own signing key +Those `X-Forwarded-*` headers are not optional and `keycloak:8080` is the +**container-internal** address on purpose. Keycloak builds the token's issuer +from them and the registry validates it against the internal address; get it +wrong and you get a 401 with an empty body. -An `upstream` row may carry a `keys` block. `bin/setup.py` does not add one -- -the mocks have no keypair -- but a real provider that signs its own catalogues -needs it, and the schema permits it. - -**Why an upstream may have keys.** The `Participant` schema declares `keys` for -every type. Its only conditional is `if type == "node" then require role and -keys`, and it has no `else` -- so that branch adds requirements for a node and -never forbids keys on an upstream. The adapter accepts such a key as a signer -too: the signature lookup filters on `participantId` alone and never compares -`type`, and `isSigning()` treats an absent `use` as signing, which matters -because this schema drops `use` and lets `alg` carry the purpose. - -**Why you would want it.** With a key on that row the provider can sign its own -catalogue and `POST /publish` straight at the **network** adapter, which -verifies the signature against the row. The provider adapter drops out of the -publish path -- and with it the unauthenticated `/publish` it otherwise has to -expose, which is the whole reason the gateway carries a deny rule for that path. - -Verified end to end: an upstream record created with an `ed25519` key signed a -catalogue and the network adapter answered `catalog/on_publish` `ACCEPTED`, -while a wrong key, a body tampered with after signing, and a missing -`Authorization` header each came back `401`. - -The header a provider has to produce: - - Signature keyId="||ed25519", - algorithm="ed25519",created="",expires="", - headers="(created) (expires) digest",signature="" - -signed over exactly this string -- real newlines, and `BLAKE-512` meaning -BLAKE2b-512, not SHA: - - (created): - (expires): - digest: BLAKE-512= - -The `osid` is the one the registry assigns the key on write, so a provider has -to read it back from a `Participant/search` after registering. - -**Add the keys when you create the record.** A partial PUT can add a `keys` -array later but cannot remove or replace one -- the registry is append-only. And -the value is bare base64 matching `^[A-Za-z0-9+/]{43}=$`, no encoding label: a -`base64:` prefix left on the front fails verification later with a decode error -that points nowhere near the registry. - -### Pointing a capability at a real API - -Two `.env` values and a re-run. To swap the weather mock for something real: - -```sh -PROVIDER_PARTICIPANT_ID=imd-mausamgram # a new id, not the mock's -MAUSAMGRAM_BASE_URL=https://the-real-api.example.gov.in -MAUSAMGRAM_PATH=/the/real/path -``` - -then `python3 bin/setup.py && docker compose up -d --force-recreate provider-adapter`. -That creates a new participant and a new binding, and re-renders the provider -config so its binding key matches. The old rows stay — see append-only below — -and become dead weight rather than a problem, since nothing sends their key. - -Things worth knowing before editing any of this: - -- **This registry is append-only.** There is no update, delete is soft, and a - soft-deleted id keeps the unique index — so an id can never be reused. Got a - row wrong? Pick a new id. This is why `PROVIDER_SUBSCRIBER_ID` and friends - are worth naming deliberately the first time. -- **Change one side of a binding key only and it fails**, in one of two ways - depending on which side. The troubleshooting section has both. -- **`path` must start with one `/` and contain no empty segment.** The schema - refuses `//get-daily`, and so does the adapter. -- **No `{"Participant": {...}}` wrapper** on a write. The registry takes the - record itself; a wrapper comes back as `extraneous key [Participant] is not - permitted`. -- **Registry schemas are read at startup.** Editing anything in - `config/registry/schemas/` needs `docker compose restart registry` before it - takes effect. +**An upstream may carry its own `keys`.** `setup.py` adds none — the mocks have +no keypair — but the schema permits it, and the adapter accepts such a key as a +signer. With one, a provider signs its own catalogue and posts `/publish` +straight at the **network** adapter, which verifies against that row. The +provider adapter drops out of the publish path, and with it the +unauthenticated `/publish` it otherwise has to expose. ## Appendix E — How a request flows -Three paths, and which adapter answers is the whole design: - ``` discover you -> exp -> network -> discovery service select you -> exp -> provider -> the upstream that owns that capability publish a catalogue system -> provider -> network -> discovery service ``` -`discover` and `publish` both end at the discovery service, and both go -through the network adapter — that adapter is what fronts discovery, verifies -the caller and re-signs. `select` never touches it: it goes straight to the -provider adapter, which calls the upstream. - -Which upstream is not in any routing table. The provider adapter runs a chain -of capability steps — `WeatherObservation`, then `MandiPrice` — and each one -builds a binding key -from the payload it is handed, serves the request if the key is its own, and -passes it along untouched if not. The step that claims it looks the upstream up -in the registry by that key. So one adapter fronts both capabilities, and a -third is a plugin plus two registry rows rather than a new route or a new -port. - -Each adapter's Beckn surface is mounted at the root, so a peer calling -`/select` lands on the module that answers select and the `baseUrl` -the registry publishes needs no path on it. There is no prefix to strip in a -gateway rule either: a proxy host forwards to a container name and port, and -the path arrives unchanged. +`discover` and `publish` both end at discovery and both go through the network +adapter, which fronts it, verifies the caller and re-signs. `select` never +touches it. + +Which upstream answers is in no routing table. The provider adapter runs a +chain of capability steps — `WeatherObservation`, then `MandiPrice` — each +building a binding key from the payload, serving the request if the key is its +own and passing it through untouched if not. So one adapter fronts both +capabilities, and a third is a plugin plus two registry rows, not a new route. **The action comes from the URL, not the payload.** The adapter strips the -module's mount path off the request path and matches what is left — `select`, -`discover` — against the routing config. The schema validator is the exception: -it reads `context.action` out of the body and ignores the path. Nothing -reconciles the two, though a mismatch usually fails validation anyway, since -two actions rarely accept the same body. +module's mount path and matches what remains — `select`, `discover` — against +the routing config. The schema validator is the exception: it reads +`context.action` from the body and ignores the path. Nothing reconciles the +two, though a mismatch usually fails validation anyway. Publishing enters at the **provider** adapter, which signs and forwards: @@ -999,69 +642,32 @@ curl -s -X POST http://127.0.0.1:9200/publish \ -H 'Content-Type: application/json' -d @your-catalog.json ``` -Three things about that: - -- **It is mounted on the exact path `/publish`,** while the Beckn surface - takes the whole subtree at `/`. Go's mux prefers the exact pattern for - `/publish` and falls back to `/` for everything else, so `/select` still - reaches the capability module. The two can coexist only because the patterns - differ — give both the same path and registration panics at startup. -- **Which is why `routing-provider.yaml` keys on an empty endpoint.** Stripping - the mount path `/publish` off the request path `/publish` leaves nothing, so - the empty string *is* the endpoint, and there is no action left for the - router to append to a target. Hence `excludeAction: true` and a target URL - written out in full. It looks odd; the alternative was posting to something - like `/internal/publish` instead, and keeping the URL the provider's - catalogue system already uses was worth more. -- **It is a second module, and has to be.** The routing step fails any action - missing from its config, so routing publish from the module that answers - select would mean listing select too — and listing select would proxy it to - the network layer instead of answering it there. -- **The catalogue body needs no `bapId` or `bppId`, and the caller need not - sign.** The provider's own catalogue system is inside its trust boundary, - so this module verifies nothing on the way in; it signs the forwarded - request as itself, and identity travels in the `Authorization` header's - `keyId` from its `keyManager` config. The network adapter verifies that - signature — and its identity check skips a body that declares no caller - rather than demanding one. - -## Appendix F — When it does not work - -Both of the common failures are a binding key disagreeing with itself, and -which 404 you get says which side is wrong. +Three things follow from that: -**404 `NET_ENTITY_NOT_FOUND`, "this module serves no capability matching the -request".** No provider step recognised the request as its own, so each passed -it through and nothing behind them answered. +- It is mounted on the **exact path** `/publish` while the Beckn surface takes + the whole subtree at `/`. Go's mux prefers the exact pattern, so `/select` + still reaches the capability module. Give both the same path and registration + panics at startup. +- **Hence `routing-provider.yaml` keys on an empty endpoint.** Stripping + `/publish` off `/publish` leaves nothing, so the empty string *is* the + endpoint — which is why `excludeAction: true` and a target URL written out in + full. +- **The body needs no `bapId`/`bppId` and the caller need not sign.** This + module verifies nothing inbound; it signs the forwarded request as itself, + and identity travels in the `Authorization` header's `keyId`. The network + adapter verifies that. -A step decides that by building a binding key from the incoming payload — the -provider id at `message.contract.commitments[].offer.provider.id` and the -capability at `...resources[].resourceAttributes.@type` — and comparing it -against the key in its own config, which `setup.py` rendered from `.env`. - -Passing through is deliberate: it is what lets this one adapter serve both -capabilities. Compare the payload against `.env`, and re-run -`bin/setup.py` plus `docker compose up -d --force-recreate provider-adapter` -after changing `.env`. - -**Every adapter exits at startup with `unrecognized step: `.** Not a -config typo. A step name that is not one of the built-ins is looked up among -the loaded plugins, and a plugin's id is the basename of its `.so` in the -image — so this is `ADAPTER_IMAGE` pointing at a build that predates the name -in the config. Check what the image actually carries: - -```sh -docker run --rm --entrypoint sh $ADAPTER_IMAGE -c 'ls plugins/*.so' -``` +## Appendix F — When it does not work -Fix the tag, do not rename the step to match an old image — the config and the -image are meant to move together. See "Updating a deployment that is already -running". +**404 `NET_ENTITY_NOT_FOUND`, "no capability matching the request".** No +provider step recognised the payload, so each passed it through and nothing +answered. A step compares a key built from the payload — provider id at +`message.contract.commitments[].offer.provider.id`, capability at +`...resources[].resourceAttributes.@type` — against its own config. Compare the +payload with `.env`, then re-run `setup.py` and recreate the adapter. -**404 naming a binding with no active record.** The other side. A step *is* -configured for the key, and it got as far as asking the registry which upstream -answers it — but there is no active `ProviderSchema` row with that -`bindingKey`, so no call plan resolves. +**404 naming a binding with no active record.** The other side: a step *is* +configured for the key, but no active `ProviderSchema` row carries it. ```sh curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema/search \ @@ -1069,178 +675,81 @@ curl -s -X POST http://127.0.0.1:8081/api/v1/ProviderSchema/search \ | python3 -c 'import json,sys; [print(r["bindingKey"], r.get("status")) for r in json.load(sys.stdin)]' ``` -Compare character for character. The registry is append-only, so a mistyped -row cannot be edited — only superseded under a new id. (This used to be a 500 -with the reason only in the log; it is a 404 that names the binding now.) - -**A 502 from a select, with an upstream status in it.** Not a binding problem: -the upstream itself answered non-2xx. The step reports 4xx immediately and -retries 5xx up to `retryMax` from the `ProviderSchema` row. Credentials are a -likely cause — the mandi mock answers 401 without a token, which is what -`MANDI_TOKEN` is for. The log line carries the redacted URL. - -**Adding a third capability**, for reference, is a plugin in the adapter image, -one more entry under `providerSteps` *and* in `steps:` in the template, and two -registry rows. Declaring a step without adding its id to `steps:` is the quiet -failure mode: it never runs, and the request passes through to the 404 above. - -**The adapters restart in a loop on the first `up`.** Expected before -`bin/setup.py` has run — there is no `config/adapters/*.yaml` yet. `make up` -sequences this correctly; a bare `docker compose up -d` does not. +Compare character for character. The registry is append-only, so a mistyped row +cannot be edited, only superseded under a new id. -**`setup.py` says the registry did not come up.** Check `make ps`. The registry -waits on Keycloak, which waits on Postgres, so a cold start takes a minute or -two — the healthcheck allows five. - -**`setup.py` says a participant is registered with a different key.** There is a -`keys/keys.json` that no longer matches the registry. Restore the old one, or -pick new `*_SUBSCRIBER_ID` values in `.env` — the old ids cannot be reused. - -**The registry refuses a write with HTTP 401 and an empty body.** The token was -minted for a different issuer than the registry validates against. Check the -`X-Forwarded-Host` header is `keycloak:8080` and not the published port. - -**A `docker compose pull` or a mapping fetch fails with "network is -unreachable".** The host advertises IPv6 but cannot route it. Add this to the -service in question: - -```yaml - sysctls: - - net.ipv6.conf.all.disable_ipv6=1 -``` - -### The gateway will not start +**`unrecognized step: ` at startup.** Not a config typo. A step name that +is not built in is looked up among loaded plugins, and a plugin's id is the +basename of its `.so` — so `ADAPTER_IMAGE` predates the config. Check what the +image carries, then fix the tag rather than the config → Appendix G. ```sh -docker compose logs nginx-proxy-manager -docker compose exec nginx-proxy-manager nginx -t -``` - -`unknown limit_req zone "exp"` means the `npm-custom` mount is missing, not -that the Advanced paste is wrong — the zone is declared in -`npm-custom/http_top.conf` and has to load before any server block that -references it. - -### 502 from a host that worked yesterday - -Almost always a recreated adapter. NPM writes a literal `proxy_pass` hostname, -which nginx resolves at reload and then caches; a `docker compose restart` of -an adapter keeps its address, but a recreate does not. - -```sh -docker compose restart nginx-proxy-manager +docker run --rm --entrypoint sh $ADAPTER_IMAGE -c 'ls plugins/*.so' ``` -The hand-written config this replaced avoided the whole failure mode by routing -every upstream through a variable and Docker's resolver. NPM generates its own -config, so that is simply the cost of the UI. - -### The certificate request fails - -In order of likelihood: - -- **Port 80 is not open to the world.** HTTP-01 validation arrives from Let's - Encrypt's servers, not from you. An SG rule scoped to your IP fails here with - a challenge timeout that reads like a DNS problem. -- **DNS does not point here yet**, or points at an address the instance lost on - its last stop/start. Check with `dig +short `, and attach an Elastic IP - if you intend to stop the VM. -- **Rate limited.** Let's Encrypt allows 5 failed validations per hostname per - hour. Once you hit it, fix the cause and wait — retrying is what keeps you - there. Use their staging environment while debugging. -- **Renewal will fail the same way in sixty days** if the DNS record or the SG - rule was only temporarily correct, and nothing will tell you at the time. - -On AWS, DNS validation with the Route 53 plugin sidesteps the first two -entirely, and is the only option for a wildcard. - -### A request through the gateway returns NPM's default page - -The `Host` header does not match any proxy host — a missing DNS record, a -typo in the domain field, or a request made against the raw IP. NPM answers -unknown hosts itself and never consults an adapter, so this says nothing about -whether the adapter is healthy. - -### 403 on /publish +**502 from a `select`, with an upstream status in it.** Not a binding problem — +the upstream answered non-2xx. 4xx is reported immediately, 5xx retried up to +`retryMax` from the ProviderSchema row. A non-2xx never reaches the mapping. -Working as intended, on every host. See "What is *not* reachable" above; the -fix is not in NPM. +**Adapters restart in a loop on the first `up`.** Expected before `setup.py` +has run. If it persists, look for a *directory* where a config file should be → +Appendix C. -### 429 on the experience host +**`setup.py` says the registry did not come up.** Check `make ps`. Keycloak's +healthcheck allows five minutes on a cold volume. -The rate limit, at 10 r/s per address with a burst of 20. A collection run that -trips it is telling you something real about the caller — but if you need -headroom for a load test, raise `rate=` in `npm-custom/http_top.conf` and -restart the gateway. +**`setup.py` says a participant is registered with a different key.** The +registry cannot update a published key and its delete is soft, so the id cannot +be reused. Restore the matching `keys/keys.json`, or pick a new +`*_SUBSCRIBER_ID` in `.env`. -### Locked out of the admin UI +**The registry refuses a write with 401 and an empty body.** The token's issuer +does not match → the `X-Forwarded-*` headers in Appendix D. -The account lives in the `npm-data` volume, and there is no reset flow. Recreate -the volume and you also lose every proxy host and certificate. Back it up: +**A pull or a mapping fetch fails with "network is unreachable".** DNS returned +an IPv6 address the host cannot route. -```sh -docker run --rm -v quick-start_npm-data:/data -v "$PWD":/backup \ - alpine tar czf /backup/npm-data.tgz -C /data . -``` +**NPM's default page, a 502 that worked yesterday, a failed certificate, or 429 +on the experience host** → Appendix B. ## Appendix G — Updating a deployment that is already running -Three things can change, and they need different work. Getting this wrong is -the most likely way to break a working VM, so the order matters. - -**Config only** — a `.tmpl`, a routing file, `.env`. Re-render and recreate: +**Config only** — a `.tmpl`, a routing file, `.env`: ```sh -make pull # git pull, and fixes the ownership NPM leaves behind -make up # step 2 re-renders the adapter configs, then recreates +make pull # git pull, and fixes the ownership NPM leaves behind +make up # step 2 re-renders the configs, then recreates ``` -`make restart` is not enough on its own for a `.tmpl` change: the adapters read -a rendered `.yaml`, and only `setup.py` writes it. +`make restart` alone is not enough for a `.tmpl` change: adapters read a +rendered `.yaml`, and only `setup.py` writes it. -**A new adapter image as well.** Any change to the plugin ids in -`config/adapters/*.tmpl` is this case, because an id is the basename of a `.so` -inside the image. The new config must not meet the old image, or every adapter -dies at startup. - -If `ADAPTER_IMAGE` names a **new tag**, set it before `make up` and that is -all. If it follows **`latest`**, `make up` alone is not enough: `pull_policy: -missing` means a tag already on disk is never re-fetched, and nothing in -`stack.sh` pulls, so the stack would quietly come back on the old image. Fetch -it explicitly first: +**A new adapter image as well.** Any change to plugin ids is this case, because +an id is a `.so` basename. If `ADAPTER_IMAGE` names a **new tag**, set it before +`make up`. If it follows **`latest`**, `make up` is not enough — +`pull_policy: missing` means a tag already on disk is never re-fetched and +nothing in `stack.sh` pulls, so the stack quietly comes back on the old image: ```sh docker compose pull provider-adapter network-adapter exp-adapter ``` -Either way, build it from the adapter repo at the commit the config expects: +Build from the adapter repo at the commit the config expects, and check the +image before deploying it — this is the step that catches a wrong branch: ```sh -git clone https://github.com/OpenAgriNet/network-adapter.git -cd network-adapter && git checkout - -docker build -f Dockerfile.adapter-with-plugins \ - --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \ - -t ghcr.io//oan-adapter:$(git rev-parse --short HEAD) . - -# the check worth doing before you push or deploy it -docker run --rm --entrypoint sh ghcr.io//oan-adapter: \ - -c 'ls plugins/ | grep -iE "weather|mandi"' +docker run --rm --entrypoint sh -c 'ls plugins/*.so' ``` -That last command should print the ids the config actually names. If it prints -something else, the image is from the wrong commit and nothing downstream will -work. +Rebuild **every** adapter image, not one. A partial rebuild presents as a config +typo in one adapter rather than a stale image in the others. -**Payload shapes changed.** If `@context` moved, catalogues already in the -discovery database still carry the old value, and `discover` matches -`schemaContext` by exact string equality — so discover alone returns zero rows -against a database seeded before the change. Run the collection top to bottom -so publish reseeds first. `updateMode: MERGE` on the same `catalogId` updates -in place rather than duplicating. +**Payload shapes changed.** If `@context` moved, catalogues already in +discovery carry the old value and `schemaContext` is matched by exact string +equality — so discover alone returns zero. Run the collection top to bottom so +publish reseeds first; `updateMode: MERGE` updates in place. -**Then check, in this order.** Cheapest first, because each failure explains -the next: +**Then check, cheapest first:** ```sh docker compose logs provider-adapter | grep 'Processor steps initialized' @@ -1248,367 +757,191 @@ docker compose logs provider-adapter | grep -iE '"level":"(error|fatal)"' make ps ``` -The first should list the capability steps by the ids the config names. The -second should be empty. Only then run the collection. +**Rolling back** is `git checkout `, `ADAPTER_IMAGE` back to the old +image, `make up` — both together, since the old image with the new config fails +at startup and the new image with the old config runs the old behaviour +silently. Note `latest` serves this badly: "the old image" has no name once the +tag has moved, so recovery is by digest. Pin a tag before a change you might +need to undo. -**Rolling back** is `git checkout `, `ADAPTER_IMAGE` back to the -old image, `make up`. Both, together — the old image with the new config fails -at startup, and the new image with the old config starts but silently runs the -old behaviour. +## Appendix H — Telemetry -Note this is the case `latest` serves badly. Rolling the config back is exact, -but "the old image" has no name if the tag has already moved, so you would be -recovering it by digest — `docker images --digests` on the VM, if it is still -there at all. Pin a tag before a change you might need to undo. +`make observability` brings up HyperDX on `127.0.0.1:8085` with OTLP on +4317/4318. It is `clickstack-local`: single-user, no team to create and no +ingestion key to mint, which is what makes it one command — and also why it +must stay on loopback, since there is no login in front of it. -## Appendix H — Telemetry +**Less arrives than the wiring suggests**, which is worth knowing before +hunting for absent traces. Discovery reads the OTLP variables but nothing in +the current build consumes them, so `OTEL_EXPORTER` stays `none`. Whether the +adapter image's SDK reads them is unverified — nothing depends on the answer, +since an absent collector makes an exporter drop spans rather than fail a +request. And container logs go nowhere near HyperDX without a collector with a +`filelog` receiver, which is not in this stack; `docker compose logs -f` +remains the way to read them. -`docker compose --profile observability up -d` brings up HyperDX on -`127.0.0.1:8085` (tunnel to reach it) with OTLP on 4317/4318. It is -`clickstack-local`, not `clickstack-all-in-one`: local runs single-user with no -team to create and no ingestion key to mint, which is what makes `up -d` the -whole setup step — and also why it must stay on loopback, since there is no -login in front of it. - -**What actually arrives today is less than the wiring suggests, and that is -worth knowing before you go looking for traces that are not there.** - -- **discovery** reads `OTEL_EXPORTER` and `OTEL_EXPORTER_OTLP_ENDPOINT` into - its config, and nothing in the current build consumes them — the only - OpenTelemetry packages in its `go.mod` are indirect. So `OTEL_EXPORTER` - stays `none` by default; setting it to `otlp` emits nothing rather than - failing. When the exporter is wired, `OTEL_EXPORTER=otlp` in `.env` is the - whole change and the endpoint already points here. -- **the three adapters** get `OTEL_EXPORTER_OTLP_ENDPOINT` and - `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`. Whether that image's SDK reads - them is unverified in either direction — the image is pulled and its source - is not in this repo. Nothing depends on the answer: an absent collector makes - an OTLP exporter drop spans, not fail a request. -- **container logs go nowhere near HyperDX** without something to ship them. - `docker compose logs -f ` remains the way to read them. Shipping - them would mean an OTel collector with a `filelog` receiver over - `/var/lib/docker/containers`, which is not in this stack. - -So treat this profile as the destination being ready and in one place, rather -than as observability that is switched on. +Treat this profile as the destination being ready, not as observability being +switched on. ## Appendix I — Schema validation -Every adapter loads the pinned Beckn v2 LTS spec and validates request bodies -against it. On the **provider adapter** a second layer runs too: it walks the -payload for objects carrying `@context` and `@type`, resolves the schema that -`@type` names, and validates the object against it. Base validation treats -`resourceAttributes` as a free-form object, so this is the only layer that -checks a capability's own attributes at all. +Every adapter validates request bodies against the pinned Beckn v2 LTS spec. On +the **provider adapter** a second layer runs too: it walks the payload for +objects carrying `@context` and `@type`, resolves the schema `@type` names, and +validates against it. Base validation treats `resourceAttributes` as free-form, +so this is the only layer checking a capability's own attributes. -The schemas are not in this repository and are not mounted. Each resource's -`@context` names the published pack, and the validator swaps `context.jsonld` -for `attributes.yaml` to fetch the schema beside it: +The schemas are neither committed nor mounted. `@context` names the published +pack and the validator swaps `context.jsonld` for `attributes.yaml`: ``` @context .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/context.jsonld fetched .../network-specs/schema-packs-v0.1/schema/MandiPrice/v0.1/attributes.yaml ``` -So a payload names the pack revision it wants to be judged against, and there is -no copy here to drift from the published one — the same reason the mappings are -fetched rather than committed. - -It is cached for 24 hours, so only the first payload after a restart pays for -the fetch. Two consequences: the provider adapter needs egress to -`raw.githubusercontent.com`, and a fetch that **fails rejects the payload** -rather than skipping validation. An `@context` on any other host is refused -before anything is fetched — `extendedSchema_allowedDomains` in the config is -the list. - -**What it does not check.** The validator library parses `if`/`then`/`else` but -never evaluates it, so every pack rule predicated on `informationMode` is -unenforced — a pass here is not full conformance to a pack. It does enforce -types, string formats, `enum`, `const`, `required`, `additionalProperties`, -`not` and `allOf`/`anyOf`/`oneOf`. - -Three consequences worth knowing before you write a payload: - -- **Each resource under a commitment needs a `quantity`.** The spec's - `Commitment.resources` requires `["id", "quantity"]` while `Resource` itself - defines no `quantity` property and the spec has no `Quantity` schema at all — - a defect upstream, not something this deployment chose. Any value satisfies - it. Without one, every `select` is refused with - `SCH_REQUIRED_FIELD_MISSING: property "quantity" is missing`. -- **A `date-time` field will not take a bare date.** `validity.startsAt` and - `endsAt` are `format: date-time` in the packs, so `2025-08-20` is refused - and `2025-08-20T00:00:00+05:30` is accepted. `arrivalDate` is `format: date` - and wants the opposite. -- **`publish` is validated, on the provider adapter.** Declaring the validator - is not enough — a plugin missing from `steps:` never runs, which is why - publish went unchecked for a while — so `validateSchema` is in that module's - `steps:` and its resources are checked against their packs like any other. - The network adapter validates nothing: its single module runs - `validateSign`, `addRoute`, `sign` and never declares a validator. - -An action the spec does not know, or a body missing a required field, comes -back as a signed NACK with a `SCH_*` code and the JSON path that failed. +So a payload names the revision it is judged against, and no copy here can +drift. Cached 24h, so only the first payload after a restart pays. Two +consequences: the adapter needs egress to `raw.githubusercontent.com`, and a +**failed fetch rejects the payload** rather than skipping validation. An +`@context` on any other host is refused before any fetch — +`extendedSchema_allowedDomains` is the list. + +**What it does not check: `if`/`then`/`else`.** The validator library parses +those keywords and never evaluates them, so every pack rule predicated on +`informationMode` is unenforced — a pass here is not pack conformance. It does +enforce types, string formats, `enum`, `const`, `required`, +`additionalProperties`, `not` and `allOf`/`anyOf`/`oneOf`. + +Three things that bite when writing a payload: + +- **Every resource under a commitment needs a `quantity`.** The spec requires + it while defining no `Quantity` schema at all — a defect upstream. Any value + satisfies it; without one every `select` is refused with + `SCH_REQUIRED_FIELD_MISSING`. +- **A `date-time` field will not take a bare date.** `validity.startsAt`/ + `endsAt` are `format: date-time`, so `2025-08-20` is refused and + `2025-08-20T00:00:00+05:30` accepted. `arrivalDate` is `format: date` and + wants the opposite. +- **`publish` is validated on the provider adapter**, because `validateSchema` + is in that module's `steps:` — declaring a validator is not enough, a plugin + missing from `steps:` never runs. The network adapter validates nothing: its + single module is `validateSign`, `addRoute`, `sign`. ## Appendix J — About the mapping files -`config/mappings/` holds the two this deployment uses — one per binding-action -— and `MAPPING_URL` and `MANDI_MAPPING_URL` point at **this repo's own copies** -over GitHub's raw CDN. So the file a reader reviews and the file the adapter -fetches are one file, and cannot drift. +`config/mappings/` holds the two this deployment uses, and `MAPPING_URL` / +`MANDI_MAPPING_URL` point at **this repo's own copies** over GitHub's raw CDN — +so the file a reader reviews and the file the adapter fetches are one file. -Each file has two halves. The request half turns the incoming Beckn payload -into the query string or body the upstream expects; the response half turns -what comes back into the resources that go in the answer. The mandi one is the -better example of why this is not a field-renaming exercise: it converts ISO -dates to the `dd-MM-yyyy` Agmarknet wants, sends `marketcode` only when the -request carried one, turns price strings into numbers, and omits a price that -was not reported rather than sending a zero. +Each has two halves: the request half turns the Beckn payload into what the +upstream expects, the response half turns the answer into resources. The mandi +one shows why this is not field renaming — ISO dates to `dd-MM-yyyy`, +`marketcode` sent only when the request carried one, price strings to numbers, +an unreported price omitted rather than sent as zero. It is a URL rather than a path because the registry publishes the full URL and -the adapter fetches it verbatim — which means a mapping has to be reachable -before it can be tested, and what this stack exercises is exactly what any -consumer fetches. - -Note the branch in those URLs. Once this merges, point them at the default -branch, or pin a tag so a deployment is not following a moving file. - -**What can be fixed here without touching code.** Quite a lot, and this is the -design intent: when a real upstream turns out to answer with different field -names, a different date format, or a nested envelope, that is a mapping edit -and a cache expiry. What is *not* fixable here is anything that depends on the -response never arriving — a non-2xx never reaches the mapping, because the -step fails first. - -To change one: edit the file here and push, or publish a fork anywhere that -serves raw text over https and put that URL in the `mappings` field of the -ProviderSchema row. The adapter caches a mapping for `cacheTTL` (one minute, -in the adapter config) and GitHub's raw CDN caches for about five, so give an -edit a few minutes to show up. +the adapter fetches it verbatim — so a mapping must be reachable before it can +be tested, and this stack exercises exactly what a consumer fetches. + +**Note the branch in those URLs.** Once this merges, point them at the default +branch or pin a tag. + +A real upstream answering with different field names, a different date format +or a nested envelope is a mapping edit and a cache expiry — no code. What is +*not* fixable here is anything depending on a response that never arrives: a +non-2xx fails the step first. Allow a few minutes for an edit to appear — +one minute of adapter cache plus about five of CDN. ## Appendix K — The layout ``` -docker-compose.yml the whole stack. Read it in tiers -- the banner - comments are the structure: registry, discovery, - adapters, observability (profile), edge (profile) +docker-compose.yml the whole stack, in tiers -- the banner comments + are the structure .env.example copy to .env -Makefile the front door: make up / up-core / down / help. +Makefile the front door; every target delegates to stack.sh bin/ bootstrap-ubuntu.sh docker and python on a fresh Ubuntu VM - stack.sh the startup order, and why it is that order. - Every make target is one line of delegation here. + stack.sh the startup order, and why it is that order setup.py keys, five registry rows, the adapter configs config/ reverse-proxy/ - npm-custom/ mounted to /data/nginx/custom, which NPM includes - http_top.conf on its own: the rate-limit zone declaration, - server_proxy.conf and the /publish deny that every proxy host gets - npm-advanced/ - exp.conf NOT loaded -- paste into the experience host's - Advanced tab. Kept here because a textarea in - NPM's database is not reviewable. - The routing table itself is not a file: it is - rows in the npm-data volume. + npm-custom/ mounted to /data/nginx/custom; NPM includes these + http_top.conf on its own -- rate-limit zone, and the /publish + server_proxy.conf deny every proxy host gets + npm-advanced/exp.conf NOT loaded. Paste into the Advanced tab; kept here + because a textarea in a database is not reviewable adapters/ experience.yaml.tmpl templates. setup.py renders these to .yaml, network.yaml.tmpl filling in the keys it generated. The rendered - provider.yaml.tmpl files hold private keys and are gitignored. - routing-experience.yaml which action goes where. it sends discover to - routing-network.yaml the network layer and select to the provider; - routing-provider.yaml provider sends publish to the network layer; - network sends discover and publish to discovery + provider.yaml.tmpl files hold private keys and are gitignored + routing-experience.yaml which action goes where: experience sends discover + routing-network.yaml to the network layer and select to the provider; + routing-provider.yaml provider sends publish to the network layer registry/ schemas/ Participant, ProviderSchema, SchemaRegistry. - Read at startup -- a change needs the registry - service restarted. + Read at startup -- a change needs a restart imports/ the Keycloak realm - discovery/ - instance.yaml.example optional override; see the compose file - mappings/ - mausamgram/ one file per binding-action: the request and the - agmarknet/ response transformation, in JSONata. These are the - files the adapters fetch over the raw CDN -- the - served copy and the reviewable copy are one file -mock-server/ - mockimd/ the two mock upstreams. Sources only: they are - mockagmarknet/ pulled as published images like everything else. - See mock-server/README.md for the build commands and - for what each deliberately gets wrong. -../postman-collection/ NOT in here -- a sibling of this directory. The - collection plus an environment file, because it is - what you point AT a stack rather than part of one -keys/keys.json generated, gitignored. The private halves of the - three adapter keypairs -- the one file here that - is worth backing up, and the reason setup.py can - be re-run without invalidating what it registered + discovery/ optional instance override + mappings/ one file per binding-action, served over the raw CDN +../postman-collection/ the collection and its environment file ``` ## Appendix L — Renaming this directory -Worth knowing before you pull a rename onto a running host, because Docker will -not warn you. +Compose takes its **project name from the directory holding the compose file**, +and every named volume is prefixed with it. So renaming this directory renames +all five volumes, and **Docker does not move the data**: a plain `make up` +afterwards starts on an empty registry, an empty catalogue, and an NPM with no +proxy hosts and no certificates. The old volumes are orphaned, not gone. -**Compose takes its project name from the directory holding the compose file**, -and every named volume is prefixed with it. So this directory becoming -`quick-start` renames all five: +`npm-letsencrypt` is the one to care about — re-issuing runs into Let's +Encrypt's duplicate limit, five per week for the same names. - docker-deployment_registry-data -> quick-start_registry-data - docker-deployment_discovery-data -> quick-start_discovery-data - docker-deployment_npm-data -> quick-start_npm-data - docker-deployment_npm-letsencrypt -> quick-start_npm-letsencrypt - docker-deployment_hyperdx-data -> quick-start_hyperdx-data - -Docker does not move data between them. A plain `make up` after the pull starts -on **empty** volumes: an empty registry, an empty discovery catalogue, and an -NPM with no proxy hosts and no certificates. The old volumes are still there, -just orphaned. - -`npm-letsencrypt` is the one to care about. Re-issuing certificates means -Let's Encrypt's duplicate-certificate limit, five per week for the same set of -names, so losing it can leave you unable to get them back for days. - -**Copy the data across before starting.** Stop the stack first, from whichever -directory name it is currently running under: +Copy the data across before starting. Stop the stack first, from whichever +name it is running under: ```sh make down - for v in registry-data discovery-data npm-data npm-letsencrypt hyperdx-data; do docker volume create "quick-start_$v" >/dev/null - docker run --rm -v "docker-deployment_$v:/from" -v "quick-start_$v:/to" alpine \ + docker run --rm -v "old-name_$v:/from" -v "quick-start_$v:/to" alpine \ sh -c 'cd /from && tar cf - . | (cd /to && tar xf -)' done ``` -Then `make up`, and check the registry has its five participants and NPM still -lists your proxy hosts before deleting anything: - -```sh -docker volume ls | grep docker-deployment_ # the old copies, once you are sure -``` - -Keycloak shares `registry-data` with the registry, so its realm travels with -that one volume -- there is nothing separate to migrate, and equally nothing -that survives if you skip it. +Then `make up`, and confirm the five participants and your proxy hosts before +deleting anything. Keycloak shares `registry-data`, so its realm travels with +that volume — and equally does not survive if you skip it. ## Appendix M — Starting over ```sh -docker compose down -v # -v also deletes the registry and discovery data -rm -rf keys config/adapters/experience.yaml config/adapters/network.yaml config/adapters/provider.yaml +docker compose down -v +rm -rf keys config/adapters/experience.yaml config/adapters/network.yaml \ + config/adapters/provider.yaml ``` -Then start again from `docker compose up -d`. New keys mean new identities, so -the provider rows have to be created again too — and the old participant ids -cannot be reused. - -## Appendix N — Before you start — the long form - -On the VM: +New keys mean new identities, so the provider rows must be created again and +**the old participant ids cannot be reused** — the registry's delete is soft +and keeps the unique index. -- Docker with Compose **v2.24 or newer**, logged in to wherever the images live - if it is private — `docker login ghcr.io`. The version floor is the - `env_file: required: false` on the HyperDX service, which is what lets an - absent `.env.docker` be absent instead of fatal. -- Python 3 and the `cryptography` package — `pip install cryptography` -- 16 GB of RAM if you run the `observability` profile — ClickHouse alone wants - 2-4 GB on top of the two JVM services. 8 GB is workable without it. +`-v` deletes **every** volume, including `npm-letsencrypt` and `npm-data` — +your certificates and your whole routing table. To clear only catalogues, drop +`quick-start_discovery-data` alone and leave the rest. -Nothing else. No external API and no tunnel: the two mock upstreams are part -of the stack, so a select has something to answer it the moment it comes up. +## Appendix N — What the collection demonstrates -`bin/bootstrap-ubuntu.sh` installs the first two on a fresh Ubuntu VM. - -## Appendix O — Testing it end to end — the long form - -**Quickest path: import `../postman-collection/`.** Six requests, 32 -assertions, nothing to fill in — publish, discover and select for each -capability, with every value already matching this deployment. There are no -registry requests: the registry has no route through the edge, so `setup.py` -seeds it instead. A green run means the stack is healthy -rather than merely answering. - -**The two `select` requests are the pair worth comparing.** They hit the same -endpoint on the same adapter, and different domain packages answer them — -because each provider step builds a binding key from the payload, serves the -request if the key is its own, and passes through anything else. Nothing routes -by URL, path or domain. That is the whole dispatch mechanism, and these two -requests are what demonstrate it. +**The two `select` requests are the pair worth comparing.** Same endpoint, same +adapter, and different domain packages answer them — because each provider step +builds a binding key from the payload, serves the request if the key is its own +and passes through anything else. Nothing routes by URL, path or domain. That +is the whole dispatch mechanism, and these two requests are what show it. **`networkAdapterUrl` is a variable no request uses, on purpose.** `discover` reaches the network adapter through the experience adapter and `publish` through the provider adapter, so nothing in the collection calls it directly. -It is listed because it is the other adapter a deployment exposes publicly: +It is listed because it is the other adapter a deployment exposes publicly — its `/publish` and `/discover` both verify signatures, so a network peer calls it directly. Signing is not something Postman does, so those calls are not -scripted — the variable is there so the address has somewhere to live, not +scripted. The variable exists to give the address somewhere to live, not because a request is missing. - -It sits at the repo root rather than in here, because it is not part of the -compose stack — it is what you point at one, and its environment file exists so -it can be aimed somewhere else. - -The rest of this section is one of those requests as curl, if you would rather -see it than run it. - -```sh -curl -s -X POST http://127.0.0.1:9202/select \ - -H 'Content-Type: application/json' \ - -d '{ - "context": { - "version": "2.0.0", "action": "select", - "networkId": "oan-dev", - "transactionId": "9f2c1a8e-4b70-4d31-9c55-6f2e0b1d7a44", - "messageId": "7d41b9e0-52a6-4c18-8b73-1e9f0a4c6d22", - "timestamp": "2026-09-04T06:12:01.330Z" - }, - "message": { "contract": { "commitments": [ { - "status": { "descriptor": { "code": "DRAFT", "name": "Draft" } }, - "resources": [ { - "id": "res:mausamgram:point-forecast", - "quantity": 1, - "resourceAttributes": { - "@context": "https://raw.githubusercontent.com/OpenAgriNet/network-specs/schema-packs-v0.1/schema/WeatherObservation/v0.1/context.jsonld", - "@type": "openagrinet:WeatherObservation", - "subjectCategories": ["Weather"], - "informationMode": "OnDemand", - "supportedObservationTypes": ["Forecast"], - "supportedParameters": ["Rainfall", "Temperature"], - "geographicGranularities": ["Point"], - "location": { "type": "Point", "coordinates": [73.7898, 19.9975] } - } - } ], - "offer": { - "id": "offer:mausamgram:open-data", - "resourceIds": ["res:mausamgram:point-forecast"], - "provider": { "id": "mausamgram-mock", - "descriptor": { "code": "IMD-NWP-01", "name": "IMD Mausamgram NWP" } } - } - } ] } } - }' | python3 -m json.tool -``` - -An `on_select` comes back with one resource per forecast day — three by -default, which is `MOCKIMD_DAYS`. - -The mandi equivalent is the same call to the same endpoint with a `MandiPrice` -resource and `agmarknet-mock` as the provider, and that is the point worth -taking from this section: **one endpoint, two capabilities, and no routing -config in between.** Each provider step builds a binding key out of the -payload it is handed, answers if the key is its own, and passes the payload -through untouched if it is not. Adding a third capability is a plugin and two -registry rows, not a new route. - -Two things about the payload: - -**No party is named, in either direction.** Identity travels in the -`Authorization` header's `keyId`, which names the signer and the key the -registry published for it; a body that declares no caller simply skips the -declared-identity comparison. Nothing needs `bapId` or `bppId`, and the `*Uri` -fields they came with were container-internal addresses that meant nothing -outside this compose network anyway. - -**The experience adapter is the only one that takes an unsigned request.** The -experience app is inside the trust boundary, so there is no network signature -to check — which is what makes this testable with a plain curl. The same call -to the provider adapter on 9200 is rejected unsigned. From 27def10d93132701ad49a12245b787f26a3027e0 Mon Sep 17 00:00:00 2001 From: ameersohel45 Date: Mon, 7 Sep 2026 17:56:39 +0530 Subject: [PATCH 78/81] docs: fix the steps -- they never said to clone the repo [OpenAgriNet/network-adapter#4] Step 1 was prerequisites and Step 2 was `cp .env.example .env`, so the walkthrough never said where the repo comes from, and every command silently assumed a working directory the reader was never put in. There is a Step 2 now that clones and cds into quick-start/, and it says so explicitly. Part 2 had the same bug, worse: Step V1 ran bin/bootstrap-ubuntu.sh from a repo that had not been cloned. The script installs git ITSELF, and documents a curl | bash form for exactly this case -- so V1 now fetches it over curl, is placed before the clone rather than after, and says to log out and back in for the docker group. The per-layer steps were also dishonest. Steps 5, 6 and 7 each ran a `docker compose up -d `, then Step 7 offered `make up-core` as an alternative that does all of it -- so the earlier commands were work the reader did not need. Bring-up is now one command in Step 4, and Steps 5-7 are what each layer is, the .env keys it uses, and the log line that confirms it. And the over-explaining is gone. Each step had two to four paragraphs of reasoning; the reasoning is what the appendices are for, so the steps carry a command, a line of what it does, and a pointer. Part 1: 190 lines to 117. Part 2: 88 to 62. Body 388 to 279, about six minutes. Dropped an ASCII annotation in the provider step that tried to bracket six config keys into pairs with box-drawing characters and did not line up, and said "those same four values" above a list of six. Verified rather than assumed: the bootstrap URL returns 200 for 5345 bytes, the branch the clone names exists on the remote, every appendix reference resolves, and the fences balance. --- quick-start/README.md | 269 +++++++++++++----------------------------- 1 file changed, 80 insertions(+), 189 deletions(-) diff --git a/quick-start/README.md b/quick-start/README.md index 4a7aad9..ab75ead 100644 --- a/quick-start/README.md +++ b/quick-start/README.md @@ -50,26 +50,28 @@ below follow startup order, so they work top to bottom. ## Step 1 — Prerequisites -- Docker with Compose v2 — `docker compose version` must work, not `docker-compose` -- `python3`, and the `cryptography` package: `pip install cryptography` +- `git` +- Docker with **Compose v2** — `docker compose version` must work, not `docker-compose` +- `python3` with `cryptography` — `pip install cryptography` - `curl` -`make up` checks all of these before starting anything, because Keycloak's -healthcheck can take five minutes on a cold volume and a missing dependency -should not surface after that wait. - -## Step 2 — Configure +## Step 2 — Get the repo ```sh -cp .env.example .env +git clone -b feat/4-docker-compose https://github.com/OpenAgriNet/helmcharts.git +cd helmcharts/quick-start ``` -Locally you need change nothing. Read the file once anyway — it is commented -where the reasoning is not obvious, and it is the reference for every key. +**Every command below runs from `quick-start/`** — it is where the compose file +and the Makefile live. -Two things to know: +## Step 3 — Configure -**The ports.** Published on `localhost` only. +```sh +cp .env.example .env +``` + +Locally, nothing in it needs changing. It publishes these on `localhost`: ``` 8081 registry 9200 provider adapter 9100 mockimd @@ -78,121 +80,51 @@ Two things to know: 8090 discovery ``` -If any of those is already taken, change it in `.env` — that is the only edit -a local run needs. The adapters reach each other by Compose service name, not -through the published ports, so moving them changes only what you type into -Postman. - -**The images.** `ADAPTER_IMAGE` and the adapter configs in this repo move -together: the configs name plugins by id, and an id is the basename of a `.so` -inside the image. A mismatch is `unrecognized step: ` at startup, which -reads like a config typo and is not. +If one is already taken, change it here — that is the only edit a local run +needs. Adapters reach each other by Compose service name, so it only changes +what you type into Postman. -## Step 3 — Start the shared services - -Everything else depends on these. +## Step 4 — Start it ```sh -docker compose up -d registry discovery +make up-core ``` -Keycloak and both databases come up with them. +Three tiers, in the only order that works: registry and discovery, then +`bin/setup.py`, then the mocks and the three adapters. Allow up to five minutes +the first time — Keycloak on a cold volume. -One key in `.env` belongs to this step: `APP_NETWORK_ID` is the network every -published catalogue is filed under, and the discovery service reads it. It is -also what a `discover` filters on, so a request naming a different network -finds nothing. - -Wait for the registry to report healthy — up to five minutes the first time, -seconds after that: - -```sh -docker compose ps registry -``` - -## Step 4 — Seed and render +`setup.py` generates a keypair per adapter, registers five participants and two +capability bindings, and renders the three adapter configs. Nothing needs +creating by hand. → Appendix C for why the order matters, Appendix D for what +it wrote. ```sh -python3 bin/setup.py +make ps ``` -One idempotent script, three jobs: - -- generates a keypair per adapter into `keys/keys.json` -- registers **five participants** — three adapters, two upstreams — and **two - capability bindings** -- renders `config/adapters/{experience,network,provider}.yaml` from the - `.tmpl` files beside them - -It reads the same `.env` that seeds the registry, which is what keeps the -registry rows and the adapter configs from disagreeing. - -Two things it is worth knowing now rather than later. It **must** run before -any adapter starts: an adapter's config is a bind-mounted *file*, and Docker -creates a *directory* at any bind-mount source that does not exist. And -`keys/keys.json` is the only copy of those keypairs — the registry cannot -update a published key, so losing the file means picking new participant ids. - -→ Appendix D for what it wrote and how to look at it. - ## Step 5 — Provider layer -Calls the upstreams and answers `select`. The only layer that talks to a -provider, and it serves both capabilities from one adapter. - -**Config it needs**, in `.env`: - -``` -PROVIDER_SUBSCRIBER_ID this adapter's own network identity -PROVIDER_PARTICIPANT_ID the weather upstream's id ─┐ each pairs with its -PROVIDER_CAPABILITY openagrinet:WeatherObservation │ *_CAPABILITY to make -MANDI_PARTICIPANT_ID the mandi upstream's id ─┤ a binding key, which -MANDI_CAPABILITY openagrinet:MandiPrice ┘ is how a step knows -MANDI_TOKEN the mandi upstream's credential its own work -``` - -Those same four values seed the registry rows, which is why changing one here -means re-running `setup.py` — and why a mismatch shows up as a 404 rather than -a config error. → Appendix F. - -`setup.py` renders these into `config/adapters/provider.yaml`. Do not edit -that file — it is regenerated, and it holds a private key. +Answers `select`, and the only layer that calls an upstream. Serves both +capabilities from one adapter. -**Bring it up** - -```sh -docker compose up -d provider-adapter -``` - -Compose starts the registry and both mocks first; it will not come up without -them. - -**Check it worked** +`.env` keys: `PROVIDER_SUBSCRIBER_ID`, and the two pairs that become binding +keys — `PROVIDER_PARTICIPANT_ID` + `PROVIDER_CAPABILITY`, +`MANDI_PARTICIPANT_ID` + `MANDI_CAPABILITY` — plus `MANDI_TOKEN`. ```sh docker compose logs provider-adapter | grep 'Processor steps initialized' ``` -You want the capability steps listed by name. +Both capability steps should be listed by name. ## Step 6 — Network layer -Fronts discovery. Verifies the caller, passes `discover` and `publish` on to -the discovery service, and re-signs as itself. - -**Config it needs** +Fronts discovery: verifies the caller, passes `discover` and `publish` on, and +re-signs as itself. -``` -NETWORK_SUBSCRIBER_ID this adapter's own network identity -``` - -**Bring it up** - -```sh -docker compose up -d network-adapter -``` - -**Check it worked** +`.env` keys: `NETWORK_SUBSCRIBER_ID`. `APP_NETWORK_ID` belongs to the discovery +service behind it — a `discover` naming a different network finds nothing. ```sh docker compose logs network-adapter | grep 'Server listening' @@ -201,138 +133,98 @@ docker compose logs network-adapter | grep 'Server listening' ## Step 7 — Experience layer The consumer's edge. Sends `discover` to the network layer and `select` -straight to the provider layer — which action goes where is -`config/adapters/routing-experience.yaml`, not a code path. - -**Config it needs** - -``` -EXP_SUBSCRIBER_ID this adapter's own network identity -``` +straight to the provider layer, per `config/adapters/routing-experience.yaml`. -**Bring it up** +`.env` keys: `EXP_SUBSCRIBER_ID`. ```sh -docker compose up -d exp-adapter +docker compose logs exp-adapter | grep 'Server listening' ``` -**Check it worked** - -```sh -make ps -``` - -All three adapters running. Or bring the whole thing up in one command, in the -order it has to happen: - -```sh -make up-core -``` +`setup.py` renders all three configs from these keys. **Do not edit the +rendered `config/adapters/*.yaml`** — they are regenerated and hold private +keys. Change `.env`, re-run `make up`. ## Step 8 — Verify end to end -Import both files from `../postman-collection/` into Postman: - -``` -api-collection.json the requests -local_postman_environment.json the URLs, already pointing at localhost -``` - -Run it top to bottom: **6 requests, 32 assertions**, two folders, one per -capability. Each folder publishes a catalogue, discovers it, then selects -against it — so run publish before discover the first time. - -A green run means the registry is seeded, both adapters sign and verify, both -mappings work, and discovery is indexing. - -Or without Postman: +Import both files from `../postman-collection/` into Postman — +`api-collection.json` and `local_postman_environment.json`, which already +points at localhost. Or: ```sh -newman run ../postman-collection/api-collection.json --folder "2. MandiPrice" +newman run ../postman-collection/api-collection.json ``` -**To point it at another deployment, edit the environment file, not the -collection.** Postman resolves an environment variable ahead of a collection -variable of the same name, so the loopback defaults stay intact for the next -person. No deployment address is committed in either file, deliberately. - -There are no registry requests in the collection either — the registry has no -route through the edge, so `setup.py` seeds it instead. +**6 requests, 32 assertions**, one folder per capability. Each folder +publishes, discovers, then selects, so run publish before discover the first +time. Green means the registry is seeded, signatures verify both ways, both +mappings work and discovery is indexing. -→ Appendix N for what the two `select` requests demonstrate, and why one -variable is deliberately called by nothing. +To point it at another deployment, edit the **environment** file, not the +collection. → Appendix N. --- # Part 2 — Run it on a VM -Part 1 Steps 2–8 apply as written. This is only what is different. +Four differences. **V1 comes before Part 1 Step 2**, because it installs `git`. ## Step V1 — Prepare the VM +Nothing is cloned yet, so fetch the script rather than running it from the repo: + ```sh -bin/bootstrap-ubuntu.sh +curl -fsSL https://raw.githubusercontent.com/OpenAgriNet/helmcharts/feat/4-docker-compose/quick-start/bin/bootstrap-ubuntu.sh | bash ``` -Docker, Compose v2, `python3-cryptography` and the docker group, idempotent. -It deliberately does not clone anything, write `.env` or start anything — -those need decisions that do not belong in a script piped from the internet. +Installs `git`, `make`, `python3-cryptography` and Docker from Docker's own apt +repo, then adds you to the `docker` group — **log out and back in** for that to +take effect. Idempotent, and it deliberately does not clone, write `.env` or +start anything. + +**8 GB** runs the stack; **16 GB** for the observability tier. -**8 GB** runs the stack. **16 GB** if you want the observability tier, which -ClickHouse alone can spend 2–4 GB on. +Then Part 1 Steps 2 and 3 as written. ## Step V2 — Change every credential -`.env.example` ships working defaults, which means they are public. Change all -of them before the VM is reachable by anyone but you: +`.env.example` ships working defaults, which means they are public: ``` POSTGRES_PASSWORD KEYCLOAK_ADMIN_PASSWORD KEYCLOAK_SECRET REGISTRY_DEFAULT_USER_PASSWORD ``` -The adapter keypairs are the exception — `setup.py` generates those, and they -are never written to `.env`. +Change all four before the VM is reachable by anyone but you. Adapter keypairs +are the exception — `setup.py` generates those and never writes them to `.env`. -## Step V3 — Bring it up +## Step V3 — Start it ```sh make up ``` -`make up` rather than `make up-core`: two more tiers. - -``` -4. nginx-proxy-manager the public edge — 80 and 443, all interfaces -5. hyperdx ClickStack. Optional, and the reason for 16 GB. -``` - -Step 4 is the one that makes the VM reachable from the internet. +`make up`, not `make up-core`: two more tiers on top of Part 1's three — +nginx-proxy-manager on 80 and 443, and HyperDX. **This is the step that makes +the VM reachable from the internet.** -## Step V4 — Expose it, and decide what is exposed +## Step V4 — Decide what is exposed -Every port except the edge's 80 and 443 is bound to `127.0.0.1`, written -literally in `docker-compose.yml` rather than taken from a variable. One -switch that moves every port to the public interface at once is a footgun; the -ports that should be reachable are reachable through the edge instead. +Everything except the edge's 80 and 443 is bound to `127.0.0.1`, written +literally in `docker-compose.yml` rather than taken from a variable. So the +registry, Keycloak and the databases are not publicly reachable — deliberately. -So the registry, Keycloak and the databases are **not** publicly reachable, -and that is deliberate — a registry whose write token any reader of `.env.example` -can mint should not be on the internet. - -Adding the proxy hosts, requesting certificates, and the `/publish` deny that -every host gets → **Appendix B**. Read it before pointing DNS at the box. +Proxy hosts, certificates, and the `/publish` deny every host gets → +**Appendix B**. Read it before pointing DNS at the box. ## Step V5 — Reach the loopback ports -From a workstation: - ```sh ssh -L 9202:127.0.0.1:9202 -L 9200:127.0.0.1:9200 \ -L 8081:127.0.0.1:8081 -L 8080:127.0.0.1:8080 -N you@the-vm ``` -The collection's defaults then work unchanged, because they already point at +The collection's defaults then work unchanged, since they already point at loopback. ## Step V6 — Observability (optional) @@ -341,9 +233,8 @@ loopback. make observability ``` -Three signals over OTLP/gRPC to HyperDX. `OTEL_ENABLED=false` builds no -exporter at all, which is what you want on a box with no collector — leaving -it true against a missing one is the noisy case. → Appendix H. +HyperDX on `127.0.0.1:8085`, OTLP on 4317/4318. → Appendix H, which is honest +about how much actually arrives. --- From 00292a2ff24ec16e3fefc719d32bfdb2493a8a1d Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:39:52 +0530 Subject: [PATCH 79/81] feat: add network-adapter chart [#2] Models the network-adapter service from docker-deployment/docker-compose.yml as a chart on oan-common. The adapter is stateless: its identity is a keypair in a Secret and everything else comes from the registry. Config is rendered by an init container from a ConfigMap of placeholders plus the identity Secret into an emptyDir, so the keypair is never written to a ConfigMap and never appears in a process environment. Renders fail outright on the five values whose absence would otherwise produce a pod that starts, reports Ready and does not work. HPA, PodDisruptionBudget, Ingress and the upstream readiness gates are all optional and off by default; ci/otel-ingress-values.yaml renders that other branch so lint covers it. --- charts/network-adapter/CHANGELOG.md | 22 ++ charts/network-adapter/Chart.yaml | 29 ++ charts/network-adapter/README.md | 110 +++++++ charts/network-adapter/ci/lint-values.yaml | 16 + .../ci/otel-ingress-values.yaml | 36 +++ .../examples/network-adapter.dev.yaml | 56 ++++ charts/network-adapter/templates/NOTES.txt | 41 +++ charts/network-adapter/templates/_helpers.tpl | 111 +++++++ .../network-adapter/templates/configmap.yaml | 128 ++++++++ .../network-adapter/templates/deployment.yaml | 186 +++++++++++ charts/network-adapter/templates/hpa.yaml | 32 ++ charts/network-adapter/templates/ingress.yaml | 44 +++ .../templates/poddisruptionbudget.yaml | 26 ++ charts/network-adapter/templates/service.yaml | 19 ++ .../templates/serviceaccount.yaml | 18 ++ charts/network-adapter/values.yaml | 298 ++++++++++++++++++ 16 files changed, 1172 insertions(+) create mode 100644 charts/network-adapter/CHANGELOG.md create mode 100644 charts/network-adapter/Chart.yaml create mode 100644 charts/network-adapter/README.md create mode 100644 charts/network-adapter/ci/lint-values.yaml create mode 100644 charts/network-adapter/ci/otel-ingress-values.yaml create mode 100644 charts/network-adapter/examples/network-adapter.dev.yaml create mode 100644 charts/network-adapter/templates/NOTES.txt create mode 100644 charts/network-adapter/templates/_helpers.tpl create mode 100644 charts/network-adapter/templates/configmap.yaml create mode 100644 charts/network-adapter/templates/deployment.yaml create mode 100644 charts/network-adapter/templates/hpa.yaml create mode 100644 charts/network-adapter/templates/ingress.yaml create mode 100644 charts/network-adapter/templates/poddisruptionbudget.yaml create mode 100644 charts/network-adapter/templates/service.yaml create mode 100644 charts/network-adapter/templates/serviceaccount.yaml create mode 100644 charts/network-adapter/values.yaml diff --git a/charts/network-adapter/CHANGELOG.md b/charts/network-adapter/CHANGELOG.md new file mode 100644 index 0000000..b4b9c54 --- /dev/null +++ b/charts/network-adapter/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +All notable changes to this chart are documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-09-04 + +### Added + +- Initial chart, modelled on the `network-adapter` service in + `docker-deployment/docker-compose.yml` (#2). +- Config rendered from a ConfigMap of placeholders plus an identity Secret, + substituted by an init container into an `emptyDir` — so the keypair is + never written to a ConfigMap and never appears in a process environment. +- Render-time failures for the five values whose absence would otherwise + produce a pod that runs, reports Ready, and does not work. +- Optional HPA, PodDisruptionBudget, Ingress and upstream readiness gates, all + off by default. diff --git a/charts/network-adapter/Chart.yaml b/charts/network-adapter/Chart.yaml new file mode 100644 index 0000000..5eb67a8 --- /dev/null +++ b/charts/network-adapter/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: network-adapter +description: >- + The OAN network-layer Beckn adapter. Verifies the caller's signature against + the registry, hands discover and publish to the discovery service, and + re-signs as itself on the way out. Stateless: its identity is a keypair held + in a Secret, and everything else it needs it reads from the registry. +type: application +version: 0.1.0 +# The adapter binary, not this chart. `/health` reports the build it is +# actually running, which is the number to trust over this one. +appVersion: "v1.9.0" +maintainers: + - name: OpenAgriNet Engineering Team + url: https://github.com/OpenAgriNet +keywords: + - oan + - openagrinet + - beckn + - adapter + - network +home: https://github.com/OpenAgriNet/helmcharts +sources: + - https://github.com/OpenAgriNet/helmcharts + - https://github.com/OpenAgriNet/network-adapter +dependencies: + - name: oan-common + version: "0.2.x" + repository: "file://../oan-common" diff --git a/charts/network-adapter/README.md b/charts/network-adapter/README.md new file mode 100644 index 0000000..a9b1907 --- /dev/null +++ b/charts/network-adapter/README.md @@ -0,0 +1,110 @@ +# network-adapter + +The network layer of the Beckn network. It answers nothing itself: it verifies +that the caller's signature checks out against the key the registry publishes +for them, forwards `discover` and `publish` to the discovery service, and signs +the response as itself. + +``` +experience adapter ─┐ + ├─signed─▶ network-adapter ──▶ discovery +provider adapter ───┘ │ + └── registry (whose key signed this?) +``` + +Stateless. Its identity is a keypair in a Secret; everything else it needs it +reads from the registry at request time. + +## Install + +```sh +helm dependency build charts/network-adapter +helm upgrade --install network-adapter charts/network-adapter -n oan \ + -f charts/network-adapter/examples/network-adapter.dev.yaml +``` + +The Secret comes first — the render fails without it. + +## What you must decide + +Five values have no default, and the chart fails rather than guessing. Each one +is something that produces a *silent* failure if wrong, which is why it is a +render error rather than a default: + +| Value | Why there is no default | +|---|---| +| `image.repository` | An empty one renders `ghcr.io/:tag`, which Helm and the API server both accept and which surfaces later as `ImagePullBackOff` | +| `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs | +| `registry.url` | The adapter verifies every caller against the registry, so with none it can verify nobody | +| `discovery.url` | Every request it accepts has nowhere to go | +| `otel.endpoint` | Only when `otel.enabled` — an enabled exporter with nowhere to send logs a failure every interval | + +## The identity Secret + +Six keys, all required: + +```sh +kubectl -n oan create secret generic network-adapter-keys \ + --from-literal=subscriberId=... \ + --from-literal=keyId=... \ + --from-literal=signingPrivateKey=... \ + --from-literal=signingPublicKey=... \ + --from-literal=encrPrivateKey=... \ + --from-literal=encrPublicKey=... +``` + +`keyId` is the **key's osid as the registry assigned it**, not a name you +choose — that is what a verifier looks the key up by. + +In the compose stack `bin/setup.py` generates these and writes them to +`keys/keys.json`. Check the field names there before scripting the command +above; that file's shape belongs to `setup.py`, not to this chart. + +### How the keys reach the process + +The adapter reads one config file and wants the keys inline in it. A plain +ConfigMap would therefore hold private keys, so instead: + +1. the **ConfigMap** holds the config with `__NETWORK_*__` placeholders — the + same shape as the `.tmpl` in the compose stack; +2. the **Secret** is mounted as files, not env vars, so the values are not + readable from `/proc//environ` of anything in the pod; +3. an **init container** substitutes one into the other and writes the result + to an `emptyDir` that dies with the pod. + +It fails loudly if a key is empty or a placeholder survives. That check exists +because the alternative is an adapter that runs, looks healthy, and produces +signatures nobody can verify — a failure that shows up in someone else's logs. + +## Key rotation needs a restart + +The keys Secret is not rendered by this chart, so its contents cannot go into +the config checksum, so changing it restarts nothing: + +```sh +kubectl -n oan rollout restart deploy/network-adapter +``` + +Changing anything else — log level, upstreams, telemetry — rolls the pods on +its own. + +## The registry row + +This chart cannot verify the half of the setup that lives outside the cluster. +The adapter's `subscriberId` needs a `Participant` row in the registry carrying +the **public** half of the keypair. Missing, or carrying a different key, and +every peer rejects what this adapter signs while the pod stays perfectly +healthy. + +## Values + +See `values.yaml` — every field is commented with what it does and what breaks +without it. `examples/network-adapter.dev.yaml` is a working dev deployment; +`ci/` holds the two files lint renders, one minimal and one with every switch +turned on. + +## Related + +- `charts/discovery` — where `discover` and `publish` are forwarded +- `charts/registry` — what signatures are verified against +- `docker-deployment/` — the compose stack this chart was modelled on diff --git a/charts/network-adapter/ci/lint-values.yaml b/charts/network-adapter/ci/lint-values.yaml new file mode 100644 index 0000000..6df6795 --- /dev/null +++ b/charts/network-adapter/ci/lint-values.yaml @@ -0,0 +1,16 @@ +# Minimum that renders. Every value here is one the chart fails without, which +# makes this file a list of what a real deployment must decide. +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "v1.9.0" + +keys: + existingSecret: + name: network-adapter-keys + +registry: + url: http://registry:8081/api/v1 + +discovery: + url: http://discovery:8080 diff --git a/charts/network-adapter/ci/otel-ingress-values.yaml b/charts/network-adapter/ci/otel-ingress-values.yaml new file mode 100644 index 0000000..9912101 --- /dev/null +++ b/charts/network-adapter/ci/otel-ingress-values.yaml @@ -0,0 +1,36 @@ +# The other side of every switch, so lint covers the branches the default +# values never reach. +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "v1.9.0" + +keys: + existingSecret: + name: network-adapter-keys + +registry: + url: http://registry:8081/api/v1 +discovery: + url: http://discovery:8080 + +otel: + enabled: true + endpoint: otel-collector.observability.svc.cluster.local:4317 + +autoscaling: + enabled: true +podDisruptionBudget: + enabled: true + +waitFor: + enabled: true + +ingress: + enabled: true + className: alb + hosts: + - host: network.oan.example.com + paths: + - path: / + pathType: Prefix diff --git a/charts/network-adapter/examples/network-adapter.dev.yaml b/charts/network-adapter/examples/network-adapter.dev.yaml new file mode 100644 index 0000000..be3496f --- /dev/null +++ b/charts/network-adapter/examples/network-adapter.dev.yaml @@ -0,0 +1,56 @@ +# network-adapter, dev. +# +# helm upgrade --install network-adapter charts/network-adapter \ +# -n oan -f charts/network-adapter/examples/network-adapter.dev.yaml +# +# The Secret comes first, and it is not optional. From what bin/setup.py wrote +# into keys/keys.json in the compose stack: +# +# kubectl -n oan create secret generic network-adapter-keys \ +# --from-literal=subscriberId="$(jq -r .network.subscriberId keys/keys.json)" \ +# --from-literal=keyId="$(jq -r .network.keyId keys/keys.json)" \ +# --from-literal=signingPrivateKey="$(jq -r .network.signingPrivate keys/keys.json)" \ +# --from-literal=signingPublicKey="$(jq -r .network.signingPublic keys/keys.json)" \ +# --from-literal=encrPrivateKey="$(jq -r .network.encrPrivate keys/keys.json)" \ +# --from-literal=encrPublicKey="$(jq -r .network.encrPublic keys/keys.json)" +# +# Check the field names against your keys.json before running that -- setup.py +# owns that shape, not this chart. + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "v1.9.0" + # The package is private. Without a docker-registry secret named here the + # deploy looks clean and the pod sits in ImagePullBackOff. + pullSecrets: [] + +keys: + existingSecret: + name: network-adapter-keys + +# In-cluster Services. These are the compose service names as they land in +# Kubernetes -- adjust to whatever your releases are actually called. +registry: + url: http://registry:8081/api/v1 + +discovery: + url: http://discovery:8080 + +logLevel: debug + +# Off until a collector exists. On with no endpoint reachable, the exporter +# logs a failure every interval and buries everything else. +otel: + enabled: false + environment: dev + +replicaCount: 1 + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/charts/network-adapter/templates/NOTES.txt b/charts/network-adapter/templates/NOTES.txt new file mode 100644 index 0000000..5c5163e --- /dev/null +++ b/charts/network-adapter/templates/NOTES.txt @@ -0,0 +1,41 @@ +{{ .Chart.Name }} {{ .Chart.Version }} — {{ include "network-adapter.fullname" . }} + + image {{ include "network-adapter.image" . }} + service {{ include "network-adapter.fullname" . }}:{{ .Values.service.port }} + registry {{ .Values.registry.url }} + discovery {{ .Values.discovery.url }} + telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} + +Check it is serving: + + kubectl -n {{ include "oan-common.namespace" . }} port-forward svc/{{ include "network-adapter.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl localhost:{{ .Values.service.port }}/health + +/health reports the build actually running, which is worth more than this +chart's appVersion. + +TWO THINGS THIS CHART CANNOT CHECK FOR YOU + +1. The Participant row. This adapter signs as {{ .Values.keys.existingSecret.name }}'s + subscriberId, and every peer verifies that signature against the public key + the registry publishes for it. If the row is missing, or carries a different + key, this pod runs and reports Ready while every peer rejects what it signs + — and the error appears in their logs, not yours. + +2. Key rotation. The keys live in Secret/{{ include "network-adapter.keysSecretName" . }}, + which this chart does not render and therefore cannot include in the config + checksum. Changing that Secret does not restart anything: + + kubectl -n {{ include "oan-common.namespace" . }} rollout restart deploy/{{ include "network-adapter.fullname" . }} + +{{- if not .Values.otel.enabled }} + +Telemetry is off, which is the right setting until a collector exists: with it +on and nothing listening the exporter logs a failure every interval. +{{- end }} +{{- if .Values.ingress.enabled }} + +Ingress is ON. This adapter verifies signatures on the way in, so what keeps +it safe is the registry's contents rather than this route — every caller must +hold a key the registry publishes for them. +{{- end }} diff --git a/charts/network-adapter/templates/_helpers.tpl b/charts/network-adapter/templates/_helpers.tpl new file mode 100644 index 0000000..6767474 --- /dev/null +++ b/charts/network-adapter/templates/_helpers.tpl @@ -0,0 +1,111 @@ +{{/* +# ============================================================================ +# NETWORK ADAPTER CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the identity, +# upstream and config-rendering wiring this adapter needs. +# ============================================================================ +*/}} + +{{- define "network-adapter.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "network-adapter.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "network-adapter.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "network-adapter.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "network-adapter.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "network-adapter.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{/* +The Secret holding this adapter's keypair. + +Failing here rather than letting the render succeed is the point. Without it +the config would carry the literal placeholders through to the process, which +starts, serves /health, reports Ready, and then fails every signature it +attempts -- at the far end, in a peer's logs, with nothing in this pod saying +why. +*/}} +{{- define "network-adapter.keysSecretName" -}} +{{- $name := .Values.keys.existingSecret.name | default "" -}} +{{- if not $name -}} +{{- fail (printf "%s: keys.existingSecret.name is required. This adapter's identity is a keypair, this chart renders no Secrets, and a pod without one starts and reports Ready while failing every signature it makes. See values.yaml for the six keys it must hold." .Chart.Name) -}} +{{- end -}} +{{- $name -}} +{{- end }} + +{{/* +The registry API base. Required for the same reason as the keys: absent, the +adapter builds a plugin pointed at nothing and every signature verification +fails on a lookup rather than on the signature. +*/}} +{{- define "network-adapter.registryUrl" -}} +{{- $url := .Values.registry.url | default "" -}} +{{- if not $url -}} +{{- fail (printf "%s: registry.url is required -- e.g. http://registry:8081/api/v1. The adapter verifies every caller against the key the registry publishes for them, so with no registry it can verify nobody." .Chart.Name) -}} +{{- end -}} +{{- $url -}} +{{- end }} + +{{/* +Where discover and publish go. No trailing path: targetType "url" appends the +action, so anything here becomes a prefix on every forwarded call. +*/}} +{{- define "network-adapter.discoveryUrl" -}} +{{- $url := .Values.discovery.url | default "" -}} +{{- if not $url -}} +{{- fail (printf "%s: discovery.url is required -- e.g. http://discovery:8080. This adapter forwards discover and publish there and answers neither itself, so with it unset every request it accepts has nowhere to go." .Chart.Name) -}} +{{- end -}} +{{- if hasSuffix "/" $url -}} +{{- fail (printf "%s: discovery.url must not end in a slash (%q). The router appends the action to it, so a trailing slash produces //discover." .Chart.Name $url) -}} +{{- end -}} +{{- $url -}} +{{- end }} + +{{/* +OTLP endpoint, required only when telemetry is on. An enabled exporter with +no endpoint dials nothing and logs a failure per export interval, which is +the noise the three enable flags exist to avoid. +*/}} +{{- define "network-adapter.otlpEndpoint" -}} +{{- if .Values.otel.enabled -}} +{{- $ep := .Values.otel.endpoint | default "" -}} +{{- if not $ep -}} +{{- fail (printf "%s: otel.endpoint is required when otel.enabled is true -- e.g. otel-collector.observability.svc.cluster.local:4317. An enabled exporter with nowhere to send logs a failure every export interval." .Chart.Name) -}} +{{- end -}} +{{- $ep -}} +{{- end -}} +{{- end }} + +{{/* +Where the rendered config lands. An emptyDir, because the init container +writes it and readOnlyRootFilesystem means there is nowhere else to write. +*/}} +{{- define "network-adapter.configDir" -}} +/app/config +{{- end }} + +{{/* +Annotation that rolls the pods when the config changes. + +The keys are not in this checksum and must not be: the Secret is not rendered +by this chart, so its content is not knowable at template time. Rotating a key +therefore needs a rollout restart -- which the NOTES print. +*/}} +{{- define "network-adapter.configChecksum" -}} +{{- include (print $.Template.BasePath "/configmap.yaml") . | sha256sum -}} +{{- end }} diff --git a/charts/network-adapter/templates/configmap.yaml b/charts/network-adapter/templates/configmap.yaml new file mode 100644 index 0000000..7f37874 --- /dev/null +++ b/charts/network-adapter/templates/configmap.yaml @@ -0,0 +1,128 @@ +{{/* +The adapter's two config files. + +adapter.yaml carries PLACEHOLDERS where the keypair goes, not the keypair -- +this is a ConfigMap, and a ConfigMap is world-readable to anything with get on +the namespace. The init container substitutes the real values from the Secret +into an emptyDir at start-up. The placeholder names match the .tmpl in the +compose stack so the two stay legible against each other. +*/}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "network-adapter.fullname" . }}-config + labels: + {{- include "network-adapter.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + adapter.yaml: | + # Rendered by the network-adapter chart. Do not edit in place: the init + # container rewrites a copy of this, and this object is replaced on every + # helm upgrade. + appName: "network-adapter" + + log: + level: {{ .Values.logLevel | quote }} + destinations: + - type: stdout + contextKeys: [transaction_id, message_id, subscriber_id, module_id] + + http: + port: {{ .Values.service.targetPort }} + timeout: + read: 30 + write: 30 + idle: 30 + + pluginManager: + root: ./plugins + + plugins: + otelsetup: + id: otelsetup + config: + serviceName: {{ .Values.otel.serviceName | quote }} + environment: {{ .Values.otel.environment | quote }} + otlpEndpoint: {{ include "network-adapter.otlpEndpoint" . | quote }} + enableMetrics: {{ .Values.otel.enabled | quote }} + enableTracing: {{ .Values.otel.enabled | quote }} + enableLogs: {{ .Values.otel.enabled | quote }} + + modules: + - name: network-adapter + # A subtree: every action lands here and the payload says which one. + path: / + handler: + type: std + # bpp because this adapter RECEIVES rather than originates. The role + # decides which declared identity validateSign would compare a signer + # against -- but no payload here declares one, so that check is + # skipped and what is verified is the signature itself, against the + # key the registry publishes for whoever signed. + role: bpp + subscriberId: __NETWORK_SUBSCRIBER_ID__ + + plugins: + registry: + id: oanregistry + config: + url: {{ include "network-adapter.registryUrl" . | quote }} + entity: {{ .Values.registry.entity | quote }} + providerEntity: {{ .Values.registry.providerEntity | quote }} + + keyManager: + id: simplekeymanager + config: + subscriberId: __NETWORK_SUBSCRIBER_ID__ + # The KEY's osid, not a friendly name: that is what the + # registry indexes keys by, and what a verifier looks up. + keyId: __NETWORK_KEY_ID__ + signingPrivateKey: "__NETWORK_SIGNING_PRIVATE__" + signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" + encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" + encrPublicKey: "__NETWORK_ENCR_PUBLIC__" + + signer: + id: signer + signValidator: + id: signvalidator + + schemaValidator: + id: schemav2validator + config: + type: url + location: {{ .Values.becknSpec.url | quote }} + cacheTTL: {{ .Values.becknSpec.cacheTTL | quote }} + extendedSchema_enabled: {{ .Values.becknSpec.extendedSchema.enabled | quote }} + extendedSchema_cacheTTL: {{ .Values.becknSpec.extendedSchema.cacheTTL | quote }} + extendedSchema_maxCacheSize: {{ .Values.becknSpec.extendedSchema.maxCacheSize | quote }} + extendedSchema_downloadTimeout: {{ .Values.becknSpec.extendedSchema.downloadTimeout | quote }} + extendedSchema_allowedDomains: {{ .Values.becknSpec.extendedSchema.allowedDomains | quote }} + + router: + id: router + config: + routingConfig: {{ include "network-adapter.configDir" . }}/routing-network.yaml + + steps: + - validateSign + - addRoute + - sign + + routing-network.yaml: | + # One job: hand the catalogue actions to the discovery service. + # + # targetType "url" appends the action to whatever base is given, so the + # base is the bare host and port with no path. + routingRules: + - version: {{ .Values.routing.version | quote }} + targetType: "url" + target: + url: {{ include "network-adapter.discoveryUrl" . | quote }} + endpoints: + {{- range .Values.routing.endpoints }} + - {{ . }} + {{- end }} diff --git a/charts/network-adapter/templates/deployment.yaml b/charts/network-adapter/templates/deployment.yaml new file mode 100644 index 0000000..c4ad2d0 --- /dev/null +++ b/charts/network-adapter/templates/deployment.yaml @@ -0,0 +1,186 @@ +apiVersion: {{ include "oan-common.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ include "network-adapter.fullname" . }} + labels: + {{- include "network-adapter.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "network-adapter.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "network-adapter.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{/* + Rolls the pods when the config changes. Deliberately does NOT cover the + keys: that Secret is not rendered here, so its content is unknowable at + template time. Rotating a key needs `kubectl rollout restart`. + */}} + checksum/config: {{ include "network-adapter.configChecksum" . }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ include "network-adapter.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- end }} + {{- with (include "oan-common.imagePullSecrets" . | trim) }} + {{- . | nindent 6 }} + {{- end }} + {{- with (include "oan-common.podSecurityContext" .) }} + securityContext: + {{- . | nindent 8 }} + {{- end }} + initContainers: + {{- include "oan-common.waitFor" (dict "ctx" . "tcp" list "http" list) | nindent 8 }} + {{/* + Substitutes the keypair into the config. + + Why an init container at all: the adapter reads one file and wants the + keys inline in it, so the alternative is a ConfigMap holding private + keys. This keeps them in a Secret, mounted as files rather than env + vars so they are not readable from /proc//environ, and writes the + result to an emptyDir that dies with the pod. + + `|` as the sed delimiter, not `/`: these values are base64 and base64 + contains `/`. It does not contain `|`. + */}} + - name: render-config + image: "{{ .Values.configRenderer.image.registry }}/{{ .Values.configRenderer.image.repository }}:{{ .Values.configRenderer.image.tag }}" + imagePullPolicy: {{ .Values.configRenderer.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -c + - | + set -eu + cp /config-template/adapter.yaml /rendered/adapter.yaml + cp /config-template/routing-network.yaml /rendered/routing-network.yaml + + subst() { + # $1 placeholder, $2 file holding the value + if [ ! -s "$2" ]; then + echo "render-config: $(basename "$2") is missing or empty in the keys Secret" >&2 + exit 1 + fi + sed -i "s|$1|$(cat "$2")|g" /rendered/adapter.yaml + } + + subst __NETWORK_SUBSCRIBER_ID__ /keys/{{ .Values.keys.existingSecret.subscriberIdKey }} + subst __NETWORK_KEY_ID__ /keys/{{ .Values.keys.existingSecret.keyIdKey }} + subst __NETWORK_SIGNING_PRIVATE__ /keys/{{ .Values.keys.existingSecret.signingPrivateKey }} + subst __NETWORK_SIGNING_PUBLIC__ /keys/{{ .Values.keys.existingSecret.signingPublicKey }} + subst __NETWORK_ENCR_PRIVATE__ /keys/{{ .Values.keys.existingSecret.encrPrivateKey }} + subst __NETWORK_ENCR_PUBLIC__ /keys/{{ .Values.keys.existingSecret.encrPublicKey }} + + # A placeholder that survived means a key was substituted with + # nothing. Catch it here rather than at the far end, in a peer's + # logs, as a signature that will not verify. + if grep -q '__NETWORK_' /rendered/adapter.yaml; then + echo "render-config: placeholders remain after substitution:" >&2 + grep -o '__NETWORK_[A-Z_]*__' /rendered/adapter.yaml | sort -u >&2 + exit 1 + fi + echo "render-config: config rendered" + volumeMounts: + - name: config-template + mountPath: /config-template + readOnly: true + - name: keys + mountPath: /keys + readOnly: true + - name: config + mountPath: /rendered + resources: + {{- toYaml .Values.configRenderer.resources | nindent 12 }} + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "network-adapter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with (include "oan-common.securityContext" .) }} + securityContext: + {{- . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + env: + {{/* + The image's CMD is `./server --config=${CONFIG_FILE}`, so this is + not decoration -- unset, the binary is handed an empty --config. + */}} + - name: CONFIG_FILE + value: {{ include "network-adapter.configDir" . }}/adapter.yaml + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.envFromSecrets }} + envFrom: + {{- range . }} + - secretRef: + name: {{ . }} + {{- end }} + {{- end }} + {{- with (include "oan-common.probes" . | trim) }} + {{- . | nindent 10 }} + {{- end }} + resources: + {{- include "oan-common.resources" . | nindent 12 }} + volumeMounts: + - name: config + mountPath: {{ include "network-adapter.configDir" . }} + readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config-template + configMap: + name: {{ include "network-adapter.fullname" . }}-config + - name: keys + secret: + secretName: {{ include "network-adapter.keysSecretName" . }} + defaultMode: 0400 + {{/* + The rendered config. emptyDir rather than anything durable: it is + derived from a ConfigMap and a Secret and is rebuilt on every start, + so persisting it would only let a stale copy outlive a key rotation. + */}} + - name: config + emptyDir: + sizeLimit: 1Mi + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/network-adapter/templates/hpa.yaml b/charts/network-adapter/templates/hpa.yaml new file mode 100644 index 0000000..477f421 --- /dev/null +++ b/charts/network-adapter/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "network-adapter.fullname" . }} + labels: + {{- include "network-adapter.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "network-adapter.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- with .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/network-adapter/templates/ingress.yaml b/charts/network-adapter/templates/ingress.yaml new file mode 100644 index 0000000..8ad236a --- /dev/null +++ b/charts/network-adapter/templates/ingress.yaml @@ -0,0 +1,44 @@ +{{- if .Values.ingress.enabled }} +{{/* +Off by default, and worth a thought before turning on. + +This adapter takes SIGNED requests -- validateSign is its first step -- so +publishing it is not the same risk as publishing an adapter that does not. But +it is still a Beckn peer endpoint, and the thing that makes it safe is that +every caller must hold a key the registry publishes. That is a property of the +registry's contents, not of this Ingress. +*/}} +apiVersion: {{ include "oan-common.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "network-adapter.fullname" . }} + labels: + {{- include "network-adapter.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType | default "Prefix" }} + backend: + service: + name: {{ include "network-adapter.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/network-adapter/templates/poddisruptionbudget.yaml b/charts/network-adapter/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..04e8bb4 --- /dev/null +++ b/charts/network-adapter/templates/poddisruptionbudget.yaml @@ -0,0 +1,26 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{/* +Both set is a config error rather than a preference: the API server rejects a +PDB carrying minAvailable and maxUnavailable together, and it does so at apply +time, which is later and less legible than here. +*/}} +{{- if and .Values.podDisruptionBudget.minAvailable .Values.podDisruptionBudget.maxUnavailable }} +{{- fail (printf "%s: set podDisruptionBudget.minAvailable or maxUnavailable, not both." .Chart.Name) }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "network-adapter.fullname" . }} + labels: + {{- include "network-adapter.labels" . | nindent 4 }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "network-adapter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/network-adapter/templates/service.yaml b/charts/network-adapter/templates/service.yaml new file mode 100644 index 0000000..cfe6ef4 --- /dev/null +++ b/charts/network-adapter/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "network-adapter.fullname" . }} + labels: + {{- include "network-adapter.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "network-adapter.selectorLabels" . | nindent 4 }} diff --git a/charts/network-adapter/templates/serviceaccount.yaml b/charts/network-adapter/templates/serviceaccount.yaml new file mode 100644 index 0000000..ad4f106 --- /dev/null +++ b/charts/network-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-adapter.serviceAccountName" . }} + labels: + {{- include "network-adapter.labels" . | nindent 4 }} + {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{/* +false by default. This adapter calls the registry and the discovery service +over HTTP and never talks to the API server, so a projected token here would +be a credential nothing uses and anything reading the filesystem could take. +*/}} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/network-adapter/values.yaml b/charts/network-adapter/values.yaml new file mode 100644 index 0000000..d182083 --- /dev/null +++ b/charts/network-adapter/values.yaml @@ -0,0 +1,298 @@ +# =========================================================================== +# network-adapter +# +# The network layer of the Beckn network. It answers nothing itself: it +# verifies that the caller's signature checks out against the key the registry +# publishes for them, forwards discover and publish to the discovery service, +# and signs the response as itself. +# +# Two things it needs that this chart cannot give it: +# +# 1. an identity -- a keypair, and the osid the registry indexes it by. That +# is `keys.existingSecret` below, and this chart renders no Secrets. +# 2. a Participant row in the registry carrying the public half. Without it +# every peer that verifies this adapter's signature fails, and the +# symptom is a signature error at the far end rather than anything here. +# =========================================================================== + +replicaCount: 1 + +# --------------------------------------------------------------------------- +# Image +# +# One image serves all three adapters -- provider, network and experience -- +# and the config decides which one a pod is. So the repository here is the +# adapter image, not a network-adapter-specific build. +# +# repository is EMPTY on purpose: the render fails while it is, rather than +# producing "ghcr.io/:v1.9.0", which Helm and the API server both accept and +# which only surfaces later as an ImagePullBackOff. Set it per environment. +# +# The runtime is cgr.dev/chainguard/wolfi-base. It has a shell -- the image's +# own CMD is `sh -c` -- so unlike the discovery service this one is not +# distroless, and the security context below is doing real work rather than +# restating what the image already guarantees. +# --------------------------------------------------------------------------- +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "latest" + digest: "" + pullPolicy: IfNotPresent + pullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + enabled: true + name: "" + annotations: {} + automountServiceAccountToken: false + +# --------------------------------------------------------------------------- +# Identity +# +# The private half of this adapter's keypair. It is a Secret, it is provided +# rather than generated, and the chart never renders one -- the same rule the +# rest of these charts follow. +# +# Create it with ExternalSecrets, or by hand from what bin/setup.py wrote into +# keys/keys.json in the compose stack: +# +# kubectl create secret generic network-adapter-keys \ +# --from-literal=subscriberId=... \ +# --from-literal=keyId=... \ +# --from-literal=signingPrivateKey=... \ +# --from-literal=signingPublicKey=... \ +# --from-literal=encrPrivateKey=... \ +# --from-literal=encrPublicKey=... +# +# keyId is the KEY's osid as the registry assigned it, not a name you choose: +# that is what a verifier looks the key up by. Getting it wrong fails at the +# far end, when a peer cannot find the key that signed. +# +# HOW IT REACHES THE PROCESS. The adapter reads one config file and wants the +# keys inline in it, so a plain ConfigMap would put private keys in an object +# that is not a Secret. Instead the ConfigMap holds the config with +# placeholders -- the same shape as the .tmpl in the compose stack -- and an +# init container substitutes the Secret into an emptyDir that only this pod +# can see. The Secret is mounted as files rather than env vars so the values +# are not readable from /proc//environ of any process in the pod. +# --------------------------------------------------------------------------- +keys: + existingSecret: + name: "" + # Key names inside that Secret. Override only if yours are named + # differently -- there is no reason to rename them otherwise. + subscriberIdKey: subscriberId + keyIdKey: keyId + signingPrivateKey: signingPrivateKey + signingPublicKey: signingPublicKey + encrPrivateKey: encrPrivateKey + encrPublicKey: encrPublicKey + +# The image that renders the config. busybox, matching what waitFor already +# uses, so a cluster that can run one can run the other. +configRenderer: + image: + registry: docker.io + repository: busybox + tag: "1.37" + pullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + +# --------------------------------------------------------------------------- +# Upstreams +# +# In-cluster addresses, and both are read at request time rather than at +# start-up -- so a registry that is briefly unreachable produces failed +# requests, not a crash loop. +# --------------------------------------------------------------------------- +registry: + # The SunbirdRC registry API base. /api/v1 included: the plugin appends the + # entity name to it. + url: "" + entity: Participant + providerEntity: ProviderSchema + +discovery: + # Where discover and publish are forwarded. No path -- the router appends + # the action, so anything here becomes a prefix on every forwarded call. + url: "" + +# --------------------------------------------------------------------------- +# Routing +# +# Both actions share one rule because they share a target: publish arrives +# from the provider adapter, discover from the experience adapter, and neither +# is answered here. +# --------------------------------------------------------------------------- +routing: + version: "2.0.0" + endpoints: + - discover + - publish + +# --------------------------------------------------------------------------- +# Beckn schema validation +# +# The base v2 schema, pinned to the LTS tag. Fetched once and cached for +# cacheTTL seconds. +# +# extendedSchema is off, and turning it on is a real decision rather than a +# stricter setting: it fetches each resource's own @context and validates +# against that, which is a network call per payload and a second thing that +# can fail on the request path. +# --------------------------------------------------------------------------- +becknSpec: + url: "https://raw.githubusercontent.com/beckn/protocol-specifications-v2/refs/tags/core-v2.0.0-lts/api/v2.0.0/beckn.yaml" + cacheTTL: "3600" + extendedSchema: + enabled: false + cacheTTL: "86400" + maxCacheSize: "100" + downloadTimeout: "30" + allowedDomains: "beckn.org,example.com,raw.githubusercontent.com" + +# --------------------------------------------------------------------------- +# Telemetry +# +# All three flags move together on purpose. With every one false the plugin +# builds no exporter and returns a no-op provider -- it does not dial, so it +# cannot log a connection failure every few seconds. Pointing a cluster with +# no collector at one is exactly how you get that, so `enabled: false` is the +# right setting until a collector exists. +# +# serviceVersion is deliberately not settable: left empty the plugin fills in +# the adapter's own build version, which is truer than anything written here +# and does not go stale. +# --------------------------------------------------------------------------- +otel: + enabled: false + # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317" + endpoint: "" + environment: dev + serviceName: oan-network-adapter + +logLevel: info + +service: + type: ClusterIP + port: 9201 + # The adapter's http.port. Changing this changes what the config tells the + # binary to listen on, not just what the Service points at. + targetPort: 9201 + annotations: {} + +ingress: + enabled: false + className: "" + annotations: {} + hosts: [] + tls: [] + +# --------------------------------------------------------------------------- +# Wait for the upstreams before starting. +# +# Off by default. The adapter reads the registry per request rather than at +# boot, so it starts happily without one and answers 5xx until it is there -- +# which is usually what you want from a rolling update. Turn it on where a +# pod that is Ready but cannot serve is worse than a pod that is slow to +# arrive. +# --------------------------------------------------------------------------- +waitFor: + enabled: false + timeoutSeconds: 300 + intervalSeconds: 3 + image: + registry: docker.io + repository: busybox + tag: "1.37" + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + +# Both hit /health, which the adapter serves from its own mux and which does +# not touch the registry or discovery -- so a readiness failure here means +# this process is unwell, not that something downstream is. +probes: + liveness: + enabled: true + path: /health + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + readiness: + enabled: true + path: /health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + +autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 6 + targetCPUUtilizationPercentage: 75 + targetMemoryUtilizationPercentage: "" + +podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + +# The image declares no USER, so it would run as root without this. 65532 is +# nobody in the distroless convention and the image has no files owned by it, +# which is fine: everything written at runtime goes to the emptyDir below. +podSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + +securityContext: + enabled: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} +podLabels: {} +podAnnotations: {} +commonLabels: {} +commonAnnotations: {} + +extraInitContainers: [] +extraVolumes: [] +extraVolumeMounts: [] +extraEnv: [] +envFromSecrets: [] From 94dd66d8002f06ccf8a1b4870e192a46347f0357 Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:40:08 +0530 Subject: [PATCH 80/81] chore: default the discovery image tag to latest [#3] The chart shipped with an empty tag, which renders an image reference with no tag at all. latest keeps the draft chart installable while the discovery service has no released version to pin to. --- charts/discovery/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/discovery/values.yaml b/charts/discovery/values.yaml index 90aa4ff..fd4aa2c 100644 --- a/charts/discovery/values.yaml +++ b/charts/discovery/values.yaml @@ -51,7 +51,7 @@ replicaCount: 1 image: registry: ghcr.io repository: openagrinet/discovery-service - tag: "" + tag: "latest" digest: "" pullPolicy: IfNotPresent pullSecrets: [] From f4fcf236a09074ebd16b9149f64cd72fe666dc5b Mon Sep 17 00:00:00 2001 From: KrutikaPhirangi <138781661+KrutikaPhirangi@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:36:21 +0530 Subject: [PATCH 81/81] feat: generalise the adapter chart to all three roles [#2] Renames network-adapter to adapter-service and makes the role a value. The provider, network and experience adapters are the same image and the same config format, so what differed between them was configuration, not three charts: role now selects the handler role, the step list and the routing target. experience is the role that justifies the abstraction. It sits inside the trust boundary and accepts unsigned requests, so it runs as bap with no validateSign step, while the other two receive from the network and must verify first. Its example sets both explicitly rather than relying on the default, so the missing step reads as deliberate. discovery.url becomes routing.rules, a list, because provider fans out to several upstreams and a single target could not express that. Every rule is checked at render time for a target url, a missing trailing slash and a non-empty endpoint list. Each example sets fullnameOverride to -adapter. Without it fullname resolves to "-adapter-service", so a release named network-adapter produced a Service called network-adapter-adapter-service and the experience adapter's route to network-adapter:9201 resolved to nothing. That only worked before because the chart itself was named network-adapter. ci/otel-ingress-values.yaml now renders the experience role, so CI covers the role branches that ci/lint-values.yaml (network) does not reach. BREAKING CHANGE: chart renamed network-adapter -> adapter-service; role is required with no default; discovery.url replaced by routing.rules; config placeholders renamed __NETWORK_* -> __ADAPTER_*; routing file is now routing-.yaml. --- README.md | 4 +- charts/adapter-service/CHANGELOG.md | 51 +++++ .../Chart.yaml | 13 +- .../README.md | 61 ++++-- charts/adapter-service/ci/lint-values.yaml | 29 +++ .../ci/otel-ingress-values.yaml | 21 +- .../adapter-service/examples/experience.yaml | 64 ++++++ charts/adapter-service/examples/network.yaml | 55 +++++ charts/adapter-service/examples/provider.yaml | 66 ++++++ charts/adapter-service/templates/NOTES.txt | 55 +++++ charts/adapter-service/templates/_helpers.tpl | 204 ++++++++++++++++++ .../templates/configmap.yaml | 93 ++++---- .../templates/deployment.yaml | 44 ++-- .../templates/hpa.yaml | 6 +- .../templates/ingress.yaml | 6 +- .../templates/poddisruptionbudget.yaml | 6 +- .../templates/service.yaml | 6 +- .../templates/serviceaccount.yaml | 4 +- .../values.yaml | 85 ++++++-- charts/network-adapter/CHANGELOG.md | 22 -- charts/network-adapter/ci/lint-values.yaml | 16 -- .../examples/network-adapter.dev.yaml | 56 ----- charts/network-adapter/templates/NOTES.txt | 41 ---- charts/network-adapter/templates/_helpers.tpl | 111 ---------- 24 files changed, 744 insertions(+), 375 deletions(-) create mode 100644 charts/adapter-service/CHANGELOG.md rename charts/{network-adapter => adapter-service}/Chart.yaml (62%) rename charts/{network-adapter => adapter-service}/README.md (57%) create mode 100644 charts/adapter-service/ci/lint-values.yaml rename charts/{network-adapter => adapter-service}/ci/otel-ingress-values.yaml (55%) create mode 100644 charts/adapter-service/examples/experience.yaml create mode 100644 charts/adapter-service/examples/network.yaml create mode 100644 charts/adapter-service/examples/provider.yaml create mode 100644 charts/adapter-service/templates/NOTES.txt create mode 100644 charts/adapter-service/templates/_helpers.tpl rename charts/{network-adapter => adapter-service}/templates/configmap.yaml (50%) rename charts/{network-adapter => adapter-service}/templates/deployment.yaml (80%) rename charts/{network-adapter => adapter-service}/templates/hpa.yaml (82%) rename charts/{network-adapter => adapter-service}/templates/ingress.yaml (88%) rename charts/{network-adapter => adapter-service}/templates/poddisruptionbudget.yaml (81%) rename charts/{network-adapter => adapter-service}/templates/service.yaml (68%) rename charts/{network-adapter => adapter-service}/templates/serviceaccount.yaml (83%) rename charts/{network-adapter => adapter-service}/values.yaml (74%) delete mode 100644 charts/network-adapter/CHANGELOG.md delete mode 100644 charts/network-adapter/ci/lint-values.yaml delete mode 100644 charts/network-adapter/examples/network-adapter.dev.yaml delete mode 100644 charts/network-adapter/templates/NOTES.txt delete mode 100644 charts/network-adapter/templates/_helpers.tpl diff --git a/README.md b/README.md index ed05c3c..9ce9ac8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Helm charts for deploying and managing OpenAgriNet (OAN) platform services. | [`keycloak`](charts/keycloak) | application | Auth for the registry, on the Sunbird RC Keycloak image. Imports the realm the registry expects. | | [`registry`](charts/registry) | application | The OAN participant registry, on Sunbird RC core. Needs `postgresql-cnpg` and `keycloak`. | | [`discovery`](charts/discovery) | application | The OAN Beckn discover-and-publish service. Needs `postgresql-cnpg` **with pgvector**. | +| [`adapter-service`](charts/adapter-service) | application | The OAN Beckn adapters. One chart, installed once per `role` — `provider`, `network` or `experience`. Needs `registry`. | ## How they fit together @@ -24,7 +25,8 @@ charts/ ├── postgresql-migration/# schema migrations (Flyway Job) ├── keycloak/ # auth for the registry ├── registry/ # the participant registry -└── discovery/ # the Beckn discover-and-publish service +├── discovery/ # the Beckn discover-and-publish service +└── adapter-service/ # the Beckn adapters — one release per role ``` Every chart depends on `oan-common` via `file://../oan-common`. diff --git a/charts/adapter-service/CHANGELOG.md b/charts/adapter-service/CHANGELOG.md new file mode 100644 index 0000000..78b9592 --- /dev/null +++ b/charts/adapter-service/CHANGELOG.md @@ -0,0 +1,51 @@ +# Changelog + +All notable changes to this chart are documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Changed + +- Renamed from `network-adapter` to `adapter-service`, and generalised from one + adapter to all three (#2). provider, network and experience run the same image and + the same config format, so `role` now selects what differed between them + rather than each needing its own chart. +- `role` is required and validated against `provider|network|experience`. It has no + default: a default would hand one adapter another one's handler role and step + list, which renders, starts, reports Ready and then mis-handles every request. +- `handler.role` and `handler.steps` derive from `role` and are overridable. + `experience` gets `bap` and `[addRoute, sign]` — it sits inside the trust boundary + and takes unsigned requests, so it has no signature to validate. The other + two get `bpp` and `[validateSign, addRoute, sign]`. +- `discovery.url` replaced by `routing.rules`, a list. `provider` fans out to + several upstreams; a single target could not express that. +- Config placeholders renamed `__NETWORK_*` to `__ADAPTER_*`, and the routing + file to `routing-.yaml`. +- `appName` and `otel.serviceName` default to `-adapter` and + `oan--adapter`, matching the compose stack's service and OTEL names. +- `http.timeout` is now a value rather than hardcoded. + +### Added + +- `examples/{provider,network,experience}.yaml` — one values file per role. +- `ci/otel-ingress-values.yaml` now renders the `experience` role, so the role + branches that `ci/lint-values.yaml` (network) does not reach are covered. +- NOTES print the role, resolved step list and routing targets, and warn that + an Ingress on the `experience` role exposes an unauthenticated entry point. + +## [0.1.0] - 2026-09-04 + +### Added + +- Initial chart, modelled on the `network-adapter` service in + `docker-deployment/docker-compose.yml` (#2). +- Config rendered from a ConfigMap of placeholders plus an identity Secret, + substituted by an init container into an `emptyDir` — so the keypair is + never written to a ConfigMap and never appears in a process environment. +- Render-time failures for the five values whose absence would otherwise + produce a pod that runs, reports Ready, and does not work. +- Optional HPA, PodDisruptionBudget, Ingress and upstream readiness gates, all + off by default. diff --git a/charts/network-adapter/Chart.yaml b/charts/adapter-service/Chart.yaml similarity index 62% rename from charts/network-adapter/Chart.yaml rename to charts/adapter-service/Chart.yaml index 5eb67a8..01b62b1 100644 --- a/charts/network-adapter/Chart.yaml +++ b/charts/adapter-service/Chart.yaml @@ -1,10 +1,11 @@ apiVersion: v2 -name: network-adapter +name: adapter-service description: >- - The OAN network-layer Beckn adapter. Verifies the caller's signature against - the registry, hands discover and publish to the discovery service, and - re-signs as itself on the way out. Stateless: its identity is a keypair held - in a Secret, and everything else it needs it reads from the registry. + One chart for the OAN Beckn adapters. provider, network and experience are the same + image and config format, so `role` selects the handler role, the step list + and the routing target; install once per role. Stateless: an adapter's + identity is a keypair held in a Secret, and everything else it reads from + the registry. type: application version: 0.1.0 # The adapter binary, not this chart. `/health` reports the build it is @@ -18,7 +19,9 @@ keywords: - openagrinet - beckn - adapter + - provider - network + - experience home: https://github.com/OpenAgriNet/helmcharts sources: - https://github.com/OpenAgriNet/helmcharts diff --git a/charts/network-adapter/README.md b/charts/adapter-service/README.md similarity index 57% rename from charts/network-adapter/README.md rename to charts/adapter-service/README.md index a9b1907..d149456 100644 --- a/charts/network-adapter/README.md +++ b/charts/adapter-service/README.md @@ -1,33 +1,57 @@ -# network-adapter +# adapter-service -The network layer of the Beckn network. It answers nothing itself: it verifies -that the caller's signature checks out against the key the registry publishes -for them, forwards `discover` and `publish` to the discovery service, and signs -the response as itself. +One chart for all three Beckn adapters. `provider`, `network` and `experience` +are the same image and the same config format, so `role` selects the handler +role, the step list and where requests are routed. ``` -experience adapter ─┐ - ├─signed─▶ network-adapter ──▶ discovery -provider adapter ───┘ │ - └── registry (whose key signed this?) + experience ──unsigned──▶ network ──▶ discovery + │ + provider ──signed─────────┤ + │ └── registry (whose key signed this?) + └──▶ mandi / agmarknet upstreams ``` -Stateless. Its identity is a keypair in a Secret; everything else it needs it +| Role | Handler | Steps | Routes to | +|---|---|---|---| +| `provider` | `bpp` | `validateSign → addRoute → sign` | upstream APIs (several) | +| `network` | `bpp` | `validateSign → addRoute → sign` | discovery | +| `experience` | `bap` | `addRoute → sign` | network | + +`experience` is the only one that accepts **unsigned** requests: the experience +app is inside the trust boundary, so there is no network signature to check. +That is why it has no `validateSign`, and why an Ingress on it exposes an +unauthenticated entry point. + +Stateless. Each adapter's identity is a keypair in a Secret; everything else it reads from the registry at request time. ## Install +Once per role. **Keep the release name per-role** — it is what the Service is +called, which is the DNS name the other adapters address. + ```sh -helm dependency build charts/network-adapter -helm upgrade --install network-adapter charts/network-adapter -n oan \ - -f charts/network-adapter/examples/network-adapter.dev.yaml +helm dependency build charts/adapter-service + +helm upgrade --install provider-adapter charts/adapter-service -n oan \ + -f charts/adapter-service/examples/provider.yaml +helm upgrade --install network-adapter charts/adapter-service -n oan \ + -f charts/adapter-service/examples/network.yaml +helm upgrade --install experience-adapter charts/adapter-service -n oan \ + -f charts/adapter-service/examples/experience.yaml ``` +Each example sets `fullnameOverride` to `-adapter`. Without it the Service +would be named `-adapter-service`, and nothing routing by name would +resolve. Rename a release and you must update whatever routes to it — `experience` +points at the network adapter, `network` points at discovery. + The Secret comes first — the render fails without it. ## What you must decide -Five values have no default, and the chart fails rather than guessing. Each one +Six values have no default, and the chart fails rather than guessing. Each one is something that produces a *silent* failure if wrong, which is why it is a render error rather than a default: @@ -36,7 +60,8 @@ render error rather than a default: | `image.repository` | An empty one renders `ghcr.io/:tag`, which Helm and the API server both accept and which surfaces later as `ImagePullBackOff` | | `keys.existingSecret.name` | Without an identity the pod starts, serves `/health`, reports Ready, and fails every signature it makes — in a peer's logs | | `registry.url` | The adapter verifies every caller against the registry, so with none it can verify nobody | -| `discovery.url` | Every request it accepts has nowhere to go | +| `role` | It decides the handler role, the step list and the routing target. A default would give one adapter another one's behaviour | +| `routing.rules` | An adapter with no route accepts requests and has nowhere to send them | | `otel.endpoint` | Only when `otel.enabled` — an enabled exporter with nowhere to send logs a failure every interval | ## The identity Secret @@ -44,7 +69,7 @@ render error rather than a default: Six keys, all required: ```sh -kubectl -n oan create secret generic network-adapter-keys \ +kubectl -n oan create secret generic -adapter-keys \ --from-literal=subscriberId=... \ --from-literal=keyId=... \ --from-literal=signingPrivateKey=... \ @@ -82,7 +107,7 @@ The keys Secret is not rendered by this chart, so its contents cannot go into the config checksum, so changing it restarts nothing: ```sh -kubectl -n oan rollout restart deploy/network-adapter +kubectl -n oan rollout restart deploy/-adapter ``` Changing anything else — log level, upstreams, telemetry — rolls the pods on @@ -99,7 +124,7 @@ healthy. ## Values See `values.yaml` — every field is commented with what it does and what breaks -without it. `examples/network-adapter.dev.yaml` is a working dev deployment; +without it. `examples/{provider,network,experience}.yaml` are working dev deployments; `ci/` holds the two files lint renders, one minimal and one with every switch turned on. diff --git a/charts/adapter-service/ci/lint-values.yaml b/charts/adapter-service/ci/lint-values.yaml new file mode 100644 index 0000000..339280d --- /dev/null +++ b/charts/adapter-service/ci/lint-values.yaml @@ -0,0 +1,29 @@ +# Minimum that renders. Every value here is one the chart fails without, which +# makes this file a list of what a real deployment must decide. +# +# role is first among them: it has no default, because a default would hand one +# adapter another one's handler role and step list. +role: network +fullnameOverride: network-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "v1.9.0" + +keys: + existingSecret: + name: network-adapter-keys + +registry: + url: http://registry:8081/api/v1 + +routing: + rules: + - version: "2.0.0" + targetType: url + target: + url: http://discovery:8080 + endpoints: + - discover + - publish diff --git a/charts/network-adapter/ci/otel-ingress-values.yaml b/charts/adapter-service/ci/otel-ingress-values.yaml similarity index 55% rename from charts/network-adapter/ci/otel-ingress-values.yaml rename to charts/adapter-service/ci/otel-ingress-values.yaml index 9912101..2eebf87 100644 --- a/charts/network-adapter/ci/otel-ingress-values.yaml +++ b/charts/adapter-service/ci/otel-ingress-values.yaml @@ -1,5 +1,12 @@ # The other side of every switch, so lint covers the branches the default # values never reach. +# +# Uses the experience role on purpose: it is the one with a different handler role and +# a shorter step list, so this case renders the role branches that +# ci/lint-values.yaml (network) does not. +role: experience +fullnameOverride: experience-adapter + image: registry: ghcr.io repository: openagrinet/network-adapter @@ -7,12 +14,20 @@ image: keys: existingSecret: - name: network-adapter-keys + name: experience-adapter-keys registry: url: http://registry:8081/api/v1 -discovery: - url: http://discovery:8080 + +routing: + rules: + - version: "2.0.0" + targetType: url + target: + url: http://network-adapter:9201 + endpoints: + - discover + - publish otel: enabled: true diff --git a/charts/adapter-service/examples/experience.yaml b/charts/adapter-service/examples/experience.yaml new file mode 100644 index 0000000..bcd15a0 --- /dev/null +++ b/charts/adapter-service/examples/experience.yaml @@ -0,0 +1,64 @@ +# =========================================================================== +# experience role -- the experience adapter, and the caller. +# +# The only adapter that takes UNSIGNED requests: the experience app is inside +# the trust boundary, so there is no network signature to check. That is what +# makes the stack testable with a plain curl, and it is why this role has no +# validateSign in its steps and runs as bap rather than bpp. +# +# It is also why the edge should rate-limit this host and nothing else. +# +# helm install experience-adapter . -f examples/experience.yaml +# +# routing.rules points at the network adapter by RELEASE name. Rename that +# release and this must change with it. +# =========================================================================== +role: experience + +# The Service name, and therefore the in-cluster DNS name other adapters use. +# Set explicitly because fullname is otherwise "-" -- release +# experience-adapter plus chart adapter-service would render +# experience-adapter-adapter-service, which is not what anything routes to. +fullnameOverride: experience-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "latest" + +keys: + existingSecret: + name: experience-adapter-keys + +registry: + url: http://registry:8081/api/v1 + +service: + port: 9202 + targetPort: 9202 + +# No validateSign: nothing that reaches this adapter carries a network +# signature. Set explicitly rather than left to the role default so the +# omission reads as deliberate. +handler: + role: bap + steps: + - addRoute + - sign + +routing: + rules: + - version: "2.0.0" + targetType: url + target: + url: http://network-adapter:9201 + endpoints: + - discover + - publish + +waitFor: + enabled: false + +otel: + enabled: false + endpoint: "" diff --git a/charts/adapter-service/examples/network.yaml b/charts/adapter-service/examples/network.yaml new file mode 100644 index 0000000..3f158e2 --- /dev/null +++ b/charts/adapter-service/examples/network.yaml @@ -0,0 +1,55 @@ +# =========================================================================== +# network role -- the network layer of the Beckn network. +# +# Verifies that the caller's signature checks out against the key the registry +# publishes for them, hands discover and publish to the discovery service, and +# signs on the way out. It answers neither action itself. +# +# helm install network-adapter . -f examples/network.yaml +# +# Keep the release name `network-adapter`: experience routes to this Service by name. +# =========================================================================== +role: network + +# The Service name, and therefore the in-cluster DNS name other adapters use. +# Set explicitly because fullname is otherwise "-" -- release +# network-adapter plus chart adapter-service would render +# network-adapter-adapter-service, which is not what anything routes to. +fullnameOverride: network-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "latest" + +keys: + existingSecret: + name: network-adapter-keys + +registry: + url: http://registry:8081/api/v1 + +service: + port: 9201 + targetPort: 9201 + +# Both actions share one rule because they share a target: publish arrives +# from the provider adapter, discover from the experience adapter, and neither +# is answered here. +routing: + rules: + - version: "2.0.0" + targetType: url + target: + url: http://discovery:8080 + endpoints: + - discover + - publish + +# The upstreams this adapter cannot serve a request without. +waitFor: + enabled: false + +otel: + enabled: false + endpoint: "" diff --git a/charts/adapter-service/examples/provider.yaml b/charts/adapter-service/examples/provider.yaml new file mode 100644 index 0000000..b567d0b --- /dev/null +++ b/charts/adapter-service/examples/provider.yaml @@ -0,0 +1,66 @@ +# =========================================================================== +# provider role -- the seller side. +# +# Receives from the network, verifies the caller, and fans the action out to +# the upstream APIs that actually answer it. Unlike the other two roles it has +# more than one target, which is why routing.rules is a list. +# +# helm install provider-adapter . -f examples/provider.yaml +# +# MANDI_TOKEN is the mandi upstream's credential. It is named here and read at +# call time, so it never lands in a config file or in the registry -- put it in +# a Secret and reference it, do not inline it. +# =========================================================================== +role: provider + +# The Service name, and therefore the in-cluster DNS name other adapters use. +# Set explicitly because fullname is otherwise "-" -- release +# provider-adapter plus chart adapter-service would render +# provider-adapter-adapter-service, which is not what anything routes to. +fullnameOverride: provider-adapter + +image: + registry: ghcr.io + repository: openagrinet/network-adapter + tag: "latest" + +keys: + existingSecret: + name: provider-adapter-keys + +registry: + url: http://registry:8081/api/v1 + +service: + port: 9200 + targetPort: 9200 + +# One rule per upstream. Each target.url is a bare host and port -- the router +# appends the action, so no path and no trailing slash. +routing: + rules: + - version: "2.0.0" + targetType: url + target: + url: http://mockimd:8080 + endpoints: + - search + - select + - version: "2.0.0" + targetType: url + target: + url: http://mockagmarknet:8080 + endpoints: + - search + +# The mandi credential. envFromSecrets pulls the whole Secret in as env vars, +# so MANDI_TOKEN is a key inside it rather than a literal here. +envFromSecrets: + - provider-adapter-upstream + +waitFor: + enabled: false + +otel: + enabled: false + endpoint: "" diff --git a/charts/adapter-service/templates/NOTES.txt b/charts/adapter-service/templates/NOTES.txt new file mode 100644 index 0000000..17ac5e6 --- /dev/null +++ b/charts/adapter-service/templates/NOTES.txt @@ -0,0 +1,55 @@ +{{ .Chart.Name }} {{ .Chart.Version }} — {{ include "adapter-service.fullname" . }} + + role {{ include "adapter-service.role" . }} ({{ include "adapter-service.handlerRole" . }}) + steps {{ include "adapter-service.steps" . | fromYamlArray | join " -> " }} + image {{ include "adapter-service.image" . }} + service {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }} + registry {{ .Values.registry.url }} + routes to {{ range $i, $r := .Values.routing.rules }}{{ if $i }}, {{ end }}{{ $r.target.url }}{{ end }} + telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} + +This Service is reachable in-cluster as {{ include "adapter-service.fullname" . }}:{{ .Values.service.port }}. +That name comes from the RELEASE name — whatever routes here must use it, so +renaming this release means updating the adapter that points at it. + +Check it is serving: + + kubectl -n {{ include "oan-common.namespace" . }} port-forward svc/{{ include "adapter-service.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + curl localhost:{{ .Values.service.port }}/health + +/health reports the build actually running, which is worth more than this +chart's appVersion. + +TWO THINGS THIS CHART CANNOT CHECK FOR YOU + +1. The Participant row. This adapter signs as {{ .Values.keys.existingSecret.name }}'s + subscriberId, and every peer verifies that signature against the public key + the registry publishes for it. If the row is missing, or carries a different + key, this pod runs and reports Ready while every peer rejects what it signs + — and the error appears in their logs, not yours. + +2. Key rotation. The keys live in Secret/{{ include "adapter-service.keysSecretName" . }}, + which this chart does not render and therefore cannot include in the config + checksum. Changing that Secret does not restart anything: + + kubectl -n {{ include "oan-common.namespace" . }} rollout restart deploy/{{ include "adapter-service.fullname" . }} + +{{- if not .Values.otel.enabled }} + +Telemetry is off, which is the right setting until a collector exists: with it +on and nothing listening the exporter logs a failure every interval. +{{- end }} +{{- if .Values.ingress.enabled }} + +Ingress is ON. +{{- if eq (include "adapter-service.role" .) "experience" }} +This is the experience role, which accepts UNSIGNED requests — it is inside the trust +boundary and has no validateSign step. An Ingress therefore exposes an +unauthenticated entry point to the network: rate-limit this host, and do not +route it from anywhere you would not trust to call the network directly. +{{- else }} +This adapter verifies signatures on the way in, so what keeps it safe is the +registry's contents rather than this route — every caller must hold a key the +registry publishes for them. +{{- end }} +{{- end }} diff --git a/charts/adapter-service/templates/_helpers.tpl b/charts/adapter-service/templates/_helpers.tpl new file mode 100644 index 0000000..db3d018 --- /dev/null +++ b/charts/adapter-service/templates/_helpers.tpl @@ -0,0 +1,204 @@ +{{/* +# ============================================================================ +# ADAPTER SERVICE CHART HELPERS +# Owner: OpenAgriNet Engineering Team +# Purpose: chart-local helpers delegating to oan-common, plus the role, +# identity, upstream and config-rendering wiring an adapter needs. +# +# One chart, three roles. provider, network and experience are the same image and the +# same config format; the role decides the handler role, the step list and +# where requests are routed. Install once per role, and keep the RELEASE name +# per-role -- the release name is what becomes the Service DNS name that the +# other adapters address. +# ============================================================================ +*/}} + +{{/* +The adapter's role: provider | network | experience. + +Required, with no default. A default would silently give one role's step list +and handler role to a different adapter -- which renders, starts, reports Ready +and then mis-handles every request, at the far end, in a peer's logs. +*/}} +{{- define "adapter-service.role" -}} +{{- $role := .Values.role | default "" -}} +{{- if not $role -}} +{{- fail (printf "%s: role is required -- one of provider, network, experience. It decides the handler role, the step list and the routing target, so there is no safe default. See examples/ for a values file per role." .Chart.Name) -}} +{{- end -}} +{{- if not (has $role (list "provider" "network" "experience")) -}} +{{- fail (printf "%s: role must be one of provider, network, experience (got %q)." .Chart.Name $role) -}} +{{- end -}} +{{- $role -}} +{{- end }} + +{{/* +The name the adapter calls itself in its config, its logs and its module. +Defaults to -adapter, matching the compose stack's service names. +*/}} +{{- define "adapter-service.appName" -}} +{{- .Values.appName | default (printf "%s-adapter" (include "adapter-service.role" .)) -}} +{{- end }} + +{{/* +OTEL service name. Defaults to oan--adapter, matching OTEL_SERVICE_NAME +in the compose stack so traces from either deployment line up. +*/}} +{{- define "adapter-service.otelServiceName" -}} +{{- .Values.otel.serviceName | default (printf "oan-%s-adapter" (include "adapter-service.role" .)) -}} +{{- end }} + +{{/* +bap originates a call, bpp receives one. experience is the caller; network and +provider receive. Overridable, because the role is a shorthand for a default +rather than a constraint. +*/}} +{{- define "adapter-service.handlerRole" -}} +{{- if .Values.handler.role -}} +{{- .Values.handler.role -}} +{{- else if eq (include "adapter-service.role" .) "experience" -}} +bap +{{- else -}} +bpp +{{- end -}} +{{- end }} + +{{/* +The step list, as a YAML array. + +experience sits inside the trust boundary and accepts unsigned requests, so it has no +signature to validate; the other two receive from the network and must verify +first. Set handler.steps to override. +*/}} +{{- define "adapter-service.steps" -}} +{{- if .Values.handler.steps -}} +{{- toYaml .Values.handler.steps -}} +{{- else if eq (include "adapter-service.role" .) "experience" -}} +{{- toYaml (list "addRoute" "sign") -}} +{{- else -}} +{{- toYaml (list "validateSign" "addRoute" "sign") -}} +{{- end -}} +{{- end }} + +{{/* +The routing config filename. One file per role so a rendered config is +self-describing when you exec into a pod. +*/}} +{{- define "adapter-service.routingFile" -}} +routing-{{ include "adapter-service.role" . }}.yaml +{{- end }} + +{{/* +The routing rules, as a YAML array. + +A list rather than one rule: provider fans out to several upstreams, while +network and experience each have a single target. Every rule needs a target url, +and a trailing slash on one produces //discover once the router appends the +action -- so that is caught here rather than at request time. +*/}} +{{- define "adapter-service.routingRules" -}} +{{- $rules := .Values.routing.rules | default list -}} +{{- if not $rules -}} +{{- fail (printf "%s: routing.rules is required and must hold at least one rule. An adapter with no route accepts requests and has nowhere to send them. See examples/%s.yaml." .Chart.Name (include "adapter-service.role" .)) -}} +{{- end -}} +{{- range $i, $rule := $rules -}} +{{- $url := $rule.target.url | default "" -}} +{{- if not $url -}} +{{- fail (printf "%s: routing.rules[%d].target.url is required -- e.g. http://discovery:8080. In-cluster this is the RELEASE name of the target, which is what its Service is called." $.Chart.Name $i) -}} +{{- end -}} +{{- if hasSuffix "/" $url -}} +{{- fail (printf "%s: routing.rules[%d].target.url must not end in a slash (%q). The router appends the action to it, so a trailing slash produces //discover." $.Chart.Name $i $url) -}} +{{- end -}} +{{- if not $rule.endpoints -}} +{{- fail (printf "%s: routing.rules[%d].endpoints is required -- e.g. [discover, publish]. A rule with no endpoints matches nothing." $.Chart.Name $i) -}} +{{- end -}} +{{- end -}} +{{- toYaml $rules -}} +{{- end }} + +{{- define "adapter-service.name" -}} +{{- include "oan-common.name" . -}} +{{- end }} + +{{- define "adapter-service.fullname" -}} +{{- include "oan-common.fullname" . -}} +{{- end }} + +{{- define "adapter-service.labels" -}} +{{- include "oan-common.labels" . -}} +{{- end }} + +{{- define "adapter-service.selectorLabels" -}} +{{- include "oan-common.selectorLabels" . -}} +{{- end }} + +{{- define "adapter-service.serviceAccountName" -}} +{{- include "oan-common.serviceAccount.name" . -}} +{{- end }} + +{{- define "adapter-service.image" -}} +{{- include "oan-common.image" . -}} +{{- end }} + +{{/* +The Secret holding this adapter's keypair. + +Failing here rather than letting the render succeed is the point. Without it +the config would carry the literal placeholders through to the process, which +starts, serves /health, reports Ready, and then fails every signature it +attempts -- at the far end, in a peer's logs, with nothing in this pod saying +why. +*/}} +{{- define "adapter-service.keysSecretName" -}} +{{- $name := .Values.keys.existingSecret.name | default "" -}} +{{- if not $name -}} +{{- fail (printf "%s: keys.existingSecret.name is required. This adapter's identity is a keypair, this chart renders no Secrets, and a pod without one starts and reports Ready while failing every signature it makes. See values.yaml for the six keys it must hold." .Chart.Name) -}} +{{- end -}} +{{- $name -}} +{{- end }} + +{{/* +The registry API base. Required for the same reason as the keys: absent, the +adapter builds a plugin pointed at nothing and every signature verification +fails on a lookup rather than on the signature. +*/}} +{{- define "adapter-service.registryUrl" -}} +{{- $url := .Values.registry.url | default "" -}} +{{- if not $url -}} +{{- fail (printf "%s: registry.url is required -- e.g. http://registry:8081/api/v1. The adapter verifies every caller against the key the registry publishes for them, so with no registry it can verify nobody." .Chart.Name) -}} +{{- end -}} +{{- $url -}} +{{- end }} + +{{/* +OTLP endpoint, required only when telemetry is on. An enabled exporter with +no endpoint dials nothing and logs a failure per export interval, which is +the noise the three enable flags exist to avoid. +*/}} +{{- define "adapter-service.otlpEndpoint" -}} +{{- if .Values.otel.enabled -}} +{{- $ep := .Values.otel.endpoint | default "" -}} +{{- if not $ep -}} +{{- fail (printf "%s: otel.endpoint is required when otel.enabled is true -- e.g. otel-collector.observability.svc.cluster.local:4317. An enabled exporter with nowhere to send logs a failure every export interval." .Chart.Name) -}} +{{- end -}} +{{- $ep -}} +{{- end -}} +{{- end }} + +{{/* +Where the rendered config lands. An emptyDir, because the init container +writes it and readOnlyRootFilesystem means there is nowhere else to write. +*/}} +{{- define "adapter-service.configDir" -}} +/app/config +{{- end }} + +{{/* +Annotation that rolls the pods when the config changes. + +The keys are not in this checksum and must not be: the Secret is not rendered +by this chart, so its content is not knowable at template time. Rotating a key +therefore needs a rollout restart -- which the NOTES print. +*/}} +{{- define "adapter-service.configChecksum" -}} +{{- include (print $.Template.BasePath "/configmap.yaml") . | sha256sum -}} +{{- end }} diff --git a/charts/network-adapter/templates/configmap.yaml b/charts/adapter-service/templates/configmap.yaml similarity index 50% rename from charts/network-adapter/templates/configmap.yaml rename to charts/adapter-service/templates/configmap.yaml index 7f37874..bda9deb 100644 --- a/charts/network-adapter/templates/configmap.yaml +++ b/charts/adapter-service/templates/configmap.yaml @@ -4,25 +4,28 @@ The adapter's two config files. adapter.yaml carries PLACEHOLDERS where the keypair goes, not the keypair -- this is a ConfigMap, and a ConfigMap is world-readable to anything with get on the namespace. The init container substitutes the real values from the Secret -into an emptyDir at start-up. The placeholder names match the .tmpl in the -compose stack so the two stay legible against each other. +into an emptyDir at start-up. + +Everything that differs between the provider, network and experience adapters is a +value: they are one image with one config format, and the role decides the +identity, the handler role, the step list and where requests are routed. */}} apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "network-adapter.fullname" . }}-config + name: {{ include "adapter-service.fullname" . }}-config labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} {{- with .Values.commonAnnotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} data: adapter.yaml: | - # Rendered by the network-adapter chart. Do not edit in place: the init - # container rewrites a copy of this, and this object is replaced on every - # helm upgrade. - appName: "network-adapter" + # Rendered by the adapter-service chart for the {{ include "adapter-service.role" . }} role. + # Do not edit in place: the init container rewrites a copy of this, and + # this object is replaced on every helm upgrade. + appName: {{ include "adapter-service.appName" . | quote }} log: level: {{ .Values.logLevel | quote }} @@ -33,9 +36,9 @@ data: http: port: {{ .Values.service.targetPort }} timeout: - read: 30 - write: 30 - idle: 30 + read: {{ .Values.http.timeout.read }} + write: {{ .Values.http.timeout.write }} + idle: {{ .Values.http.timeout.idle }} pluginManager: root: ./plugins @@ -44,46 +47,47 @@ data: otelsetup: id: otelsetup config: - serviceName: {{ .Values.otel.serviceName | quote }} + serviceName: {{ include "adapter-service.otelServiceName" . | quote }} environment: {{ .Values.otel.environment | quote }} - otlpEndpoint: {{ include "network-adapter.otlpEndpoint" . | quote }} + otlpEndpoint: {{ include "adapter-service.otlpEndpoint" . | quote }} enableMetrics: {{ .Values.otel.enabled | quote }} enableTracing: {{ .Values.otel.enabled | quote }} enableLogs: {{ .Values.otel.enabled | quote }} modules: - - name: network-adapter + - name: {{ include "adapter-service.appName" . | quote }} # A subtree: every action lands here and the payload says which one. - path: / + path: {{ .Values.handler.path | quote }} handler: type: std - # bpp because this adapter RECEIVES rather than originates. The role - # decides which declared identity validateSign would compare a signer - # against -- but no payload here declares one, so that check is - # skipped and what is verified is the signature itself, against the - # key the registry publishes for whoever signed. - role: bpp - subscriberId: __NETWORK_SUBSCRIBER_ID__ + {{/* + bap originates a call, bpp receives one. It decides which declared + identity validateSign would compare a signer against -- so it is not + cosmetic, and it is why experience (which originates) differs from network + and provider (which receive). + */}} + role: {{ include "adapter-service.handlerRole" . | quote }} + subscriberId: __ADAPTER_SUBSCRIBER_ID__ plugins: registry: id: oanregistry config: - url: {{ include "network-adapter.registryUrl" . | quote }} + url: {{ include "adapter-service.registryUrl" . | quote }} entity: {{ .Values.registry.entity | quote }} providerEntity: {{ .Values.registry.providerEntity | quote }} keyManager: id: simplekeymanager config: - subscriberId: __NETWORK_SUBSCRIBER_ID__ + subscriberId: __ADAPTER_SUBSCRIBER_ID__ # The KEY's osid, not a friendly name: that is what the # registry indexes keys by, and what a verifier looks up. - keyId: __NETWORK_KEY_ID__ - signingPrivateKey: "__NETWORK_SIGNING_PRIVATE__" - signingPublicKey: "__NETWORK_SIGNING_PUBLIC__" - encrPrivateKey: "__NETWORK_ENCR_PRIVATE__" - encrPublicKey: "__NETWORK_ENCR_PUBLIC__" + keyId: __ADAPTER_KEY_ID__ + signingPrivateKey: "__ADAPTER_SIGNING_PRIVATE__" + signingPublicKey: "__ADAPTER_SIGNING_PUBLIC__" + encrPrivateKey: "__ADAPTER_ENCR_PRIVATE__" + encrPublicKey: "__ADAPTER_ENCR_PUBLIC__" signer: id: signer @@ -105,24 +109,23 @@ data: router: id: router config: - routingConfig: {{ include "network-adapter.configDir" . }}/routing-network.yaml + routingConfig: {{ include "adapter-service.configDir" . }}/{{ include "adapter-service.routingFile" . }} + {{/* + The step list is role-specific and the difference matters. experience is + inside the trust boundary and takes unsigned requests, so it has no + validateSign to run; network and provider receive from the network + and must verify before they do anything else. + */}} steps: - - validateSign - - addRoute - - sign + {{- range include "adapter-service.steps" . | fromYamlArray }} + - {{ . }} + {{- end }} - routing-network.yaml: | - # One job: hand the catalogue actions to the discovery service. + {{ include "adapter-service.routingFile" . }}: | + # Where this adapter hands requests on. # - # targetType "url" appends the action to whatever base is given, so the - # base is the bare host and port with no path. + # targetType "url" appends the action to whatever base is given, so a base + # must be the bare host and port with no path. routingRules: - - version: {{ .Values.routing.version | quote }} - targetType: "url" - target: - url: {{ include "network-adapter.discoveryUrl" . | quote }} - endpoints: - {{- range .Values.routing.endpoints }} - - {{ . }} - {{- end }} + {{- include "adapter-service.routingRules" . | nindent 6 }} diff --git a/charts/network-adapter/templates/deployment.yaml b/charts/adapter-service/templates/deployment.yaml similarity index 80% rename from charts/network-adapter/templates/deployment.yaml rename to charts/adapter-service/templates/deployment.yaml index c4ad2d0..52505bf 100644 --- a/charts/network-adapter/templates/deployment.yaml +++ b/charts/adapter-service/templates/deployment.yaml @@ -1,9 +1,9 @@ apiVersion: {{ include "oan-common.deployment.apiVersion" . }} kind: Deployment metadata: - name: {{ include "network-adapter.fullname" . }} + name: {{ include "adapter-service.fullname" . }} labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} {{- with .Values.commonAnnotations }} annotations: {{- toYaml . | nindent 4 }} @@ -14,11 +14,11 @@ spec: {{- end }} selector: matchLabels: - {{- include "network-adapter.selectorLabels" . | nindent 6 }} + {{- include "adapter-service.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "network-adapter.selectorLabels" . | nindent 8 }} + {{- include "adapter-service.selectorLabels" . | nindent 8 }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -28,13 +28,13 @@ spec: keys: that Secret is not rendered here, so its content is unknowable at template time. Rotating a key needs `kubectl rollout restart`. */}} - checksum/config: {{ include "network-adapter.configChecksum" . }} + checksum/config: {{ include "adapter-service.configChecksum" . }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} spec: {{- if .Values.serviceAccount.enabled }} - serviceAccountName: {{ include "network-adapter.serviceAccountName" . }} + serviceAccountName: {{ include "adapter-service.serviceAccountName" . }} automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} {{- end }} {{- with (include "oan-common.imagePullSecrets" . | trim) }} @@ -70,8 +70,8 @@ spec: - -c - | set -eu - cp /config-template/adapter.yaml /rendered/adapter.yaml - cp /config-template/routing-network.yaml /rendered/routing-network.yaml + cp /config-template/adapter.yaml /rendered/adapter.yaml + cp /config-template/{{ include "adapter-service.routingFile" . }} /rendered/{{ include "adapter-service.routingFile" . }} subst() { # $1 placeholder, $2 file holding the value @@ -82,22 +82,22 @@ spec: sed -i "s|$1|$(cat "$2")|g" /rendered/adapter.yaml } - subst __NETWORK_SUBSCRIBER_ID__ /keys/{{ .Values.keys.existingSecret.subscriberIdKey }} - subst __NETWORK_KEY_ID__ /keys/{{ .Values.keys.existingSecret.keyIdKey }} - subst __NETWORK_SIGNING_PRIVATE__ /keys/{{ .Values.keys.existingSecret.signingPrivateKey }} - subst __NETWORK_SIGNING_PUBLIC__ /keys/{{ .Values.keys.existingSecret.signingPublicKey }} - subst __NETWORK_ENCR_PRIVATE__ /keys/{{ .Values.keys.existingSecret.encrPrivateKey }} - subst __NETWORK_ENCR_PUBLIC__ /keys/{{ .Values.keys.existingSecret.encrPublicKey }} + subst __ADAPTER_SUBSCRIBER_ID__ /keys/{{ .Values.keys.existingSecret.subscriberIdKey }} + subst __ADAPTER_KEY_ID__ /keys/{{ .Values.keys.existingSecret.keyIdKey }} + subst __ADAPTER_SIGNING_PRIVATE__ /keys/{{ .Values.keys.existingSecret.signingPrivateKey }} + subst __ADAPTER_SIGNING_PUBLIC__ /keys/{{ .Values.keys.existingSecret.signingPublicKey }} + subst __ADAPTER_ENCR_PRIVATE__ /keys/{{ .Values.keys.existingSecret.encrPrivateKey }} + subst __ADAPTER_ENCR_PUBLIC__ /keys/{{ .Values.keys.existingSecret.encrPublicKey }} # A placeholder that survived means a key was substituted with # nothing. Catch it here rather than at the far end, in a peer's # logs, as a signature that will not verify. - if grep -q '__NETWORK_' /rendered/adapter.yaml; then + if grep -q '__ADAPTER_' /rendered/adapter.yaml; then echo "render-config: placeholders remain after substitution:" >&2 - grep -o '__NETWORK_[A-Z_]*__' /rendered/adapter.yaml | sort -u >&2 + grep -o '__ADAPTER_[A-Z_]*__' /rendered/adapter.yaml | sort -u >&2 exit 1 fi - echo "render-config: config rendered" + echo "render-config: config rendered for the {{ include "adapter-service.role" . }} role" volumeMounts: - name: config-template mountPath: /config-template @@ -114,7 +114,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }} - image: {{ include "network-adapter.image" . }} + image: {{ include "adapter-service.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with (include "oan-common.securityContext" .) }} securityContext: @@ -130,7 +130,7 @@ spec: not decoration -- unset, the binary is handed an empty --config. */}} - name: CONFIG_FILE - value: {{ include "network-adapter.configDir" . }}/adapter.yaml + value: {{ include "adapter-service.configDir" . }}/adapter.yaml {{- with .Values.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} @@ -148,7 +148,7 @@ spec: {{- include "oan-common.resources" . | nindent 12 }} volumeMounts: - name: config - mountPath: {{ include "network-adapter.configDir" . }} + mountPath: {{ include "adapter-service.configDir" . }} readOnly: true {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} @@ -156,10 +156,10 @@ spec: volumes: - name: config-template configMap: - name: {{ include "network-adapter.fullname" . }}-config + name: {{ include "adapter-service.fullname" . }}-config - name: keys secret: - secretName: {{ include "network-adapter.keysSecretName" . }} + secretName: {{ include "adapter-service.keysSecretName" . }} defaultMode: 0400 {{/* The rendered config. emptyDir rather than anything durable: it is diff --git a/charts/network-adapter/templates/hpa.yaml b/charts/adapter-service/templates/hpa.yaml similarity index 82% rename from charts/network-adapter/templates/hpa.yaml rename to charts/adapter-service/templates/hpa.yaml index 477f421..6f69699 100644 --- a/charts/network-adapter/templates/hpa.yaml +++ b/charts/adapter-service/templates/hpa.yaml @@ -2,14 +2,14 @@ apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: {{ include "network-adapter.fullname" . }} + name: {{ include "adapter-service.fullname" . }} labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "network-adapter.fullname" . }} + name: {{ include "adapter-service.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} metrics: diff --git a/charts/network-adapter/templates/ingress.yaml b/charts/adapter-service/templates/ingress.yaml similarity index 88% rename from charts/network-adapter/templates/ingress.yaml rename to charts/adapter-service/templates/ingress.yaml index 8ad236a..8546a11 100644 --- a/charts/network-adapter/templates/ingress.yaml +++ b/charts/adapter-service/templates/ingress.yaml @@ -11,9 +11,9 @@ registry's contents, not of this Ingress. apiVersion: {{ include "oan-common.ingress.apiVersion" . }} kind: Ingress metadata: - name: {{ include "network-adapter.fullname" . }} + name: {{ include "adapter-service.fullname" . }} labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} {{- with (merge (dict) .Values.ingress.annotations (.Values.commonAnnotations | default dict)) }} annotations: {{- toYaml . | nindent 4 }} @@ -36,7 +36,7 @@ spec: pathType: {{ .pathType | default "Prefix" }} backend: service: - name: {{ include "network-adapter.fullname" $ }} + name: {{ include "adapter-service.fullname" $ }} port: number: {{ $.Values.service.port }} {{- end }} diff --git a/charts/network-adapter/templates/poddisruptionbudget.yaml b/charts/adapter-service/templates/poddisruptionbudget.yaml similarity index 81% rename from charts/network-adapter/templates/poddisruptionbudget.yaml rename to charts/adapter-service/templates/poddisruptionbudget.yaml index 04e8bb4..16f298f 100644 --- a/charts/network-adapter/templates/poddisruptionbudget.yaml +++ b/charts/adapter-service/templates/poddisruptionbudget.yaml @@ -10,9 +10,9 @@ time, which is later and less legible than here. apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: {{ include "network-adapter.fullname" . }} + name: {{ include "adapter-service.fullname" . }} labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} spec: {{- with .Values.podDisruptionBudget.minAvailable }} minAvailable: {{ . }} @@ -22,5 +22,5 @@ spec: {{- end }} selector: matchLabels: - {{- include "network-adapter.selectorLabels" . | nindent 6 }} + {{- include "adapter-service.selectorLabels" . | nindent 6 }} {{- end }} diff --git a/charts/network-adapter/templates/service.yaml b/charts/adapter-service/templates/service.yaml similarity index 68% rename from charts/network-adapter/templates/service.yaml rename to charts/adapter-service/templates/service.yaml index cfe6ef4..35e76ea 100644 --- a/charts/network-adapter/templates/service.yaml +++ b/charts/adapter-service/templates/service.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "network-adapter.fullname" . }} + name: {{ include "adapter-service.fullname" . }} labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} {{- with (merge (dict) .Values.service.annotations (.Values.commonAnnotations | default dict)) }} annotations: {{- toYaml . | nindent 4 }} @@ -16,4 +16,4 @@ spec: protocol: TCP name: http selector: - {{- include "network-adapter.selectorLabels" . | nindent 4 }} + {{- include "adapter-service.selectorLabels" . | nindent 4 }} diff --git a/charts/network-adapter/templates/serviceaccount.yaml b/charts/adapter-service/templates/serviceaccount.yaml similarity index 83% rename from charts/network-adapter/templates/serviceaccount.yaml rename to charts/adapter-service/templates/serviceaccount.yaml index ad4f106..0464530 100644 --- a/charts/network-adapter/templates/serviceaccount.yaml +++ b/charts/adapter-service/templates/serviceaccount.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "network-adapter.serviceAccountName" . }} + name: {{ include "adapter-service.serviceAccountName" . }} labels: - {{- include "network-adapter.labels" . | nindent 4 }} + {{- include "adapter-service.labels" . | nindent 4 }} {{- with (merge (dict) .Values.serviceAccount.annotations (.Values.commonAnnotations | default dict)) }} annotations: {{- toYaml . | nindent 4 }} diff --git a/charts/network-adapter/values.yaml b/charts/adapter-service/values.yaml similarity index 74% rename from charts/network-adapter/values.yaml rename to charts/adapter-service/values.yaml index d182083..3447c21 100644 --- a/charts/network-adapter/values.yaml +++ b/charts/adapter-service/values.yaml @@ -1,12 +1,21 @@ # =========================================================================== -# network-adapter +# adapter-service # -# The network layer of the Beckn network. It answers nothing itself: it -# verifies that the caller's signature checks out against the key the registry -# publishes for them, forwards discover and publish to the discovery service, -# and signs the response as itself. +# One chart for all three Beckn adapters. provider, network and experience are the +# same image and the same config format; `role` below decides the handler +# role, the step list and where requests are routed. # -# Two things it needs that this chart cannot give it: +# INSTALL ONCE PER ROLE, and keep the RELEASE name per-role: +# +# helm install provider-adapter . -f examples/provider.yaml +# helm install network-adapter . -f examples/network.yaml +# helm install experience-adapter . -f examples/experience.yaml +# +# The release name is what the Service is called, which is the DNS name the +# other adapters address. Rename a release and you must update whatever routes +# to it -- experience routes to the network adapter, provider is routed to by network. +# +# Two things every role needs that this chart cannot give it: # # 1. an identity -- a keypair, and the osid the registry indexes it by. That # is `keys.existingSecret` below, and this chart renders no Secrets. @@ -15,6 +24,18 @@ # symptom is a signature error at the far end rather than anything here. # =========================================================================== +# --------------------------------------------------------------------------- +# Role: provider | network | experience +# +# Required, with no default. A default would hand one role's handler role and +# step list to a different adapter, which renders, starts, reports Ready and +# then mis-handles every request. See examples/ for a values file per role. +# --------------------------------------------------------------------------- +role: "" + +# Overrides for what the role implies. Leave empty to take the role's default. +appName: "" + replicaCount: 1 # --------------------------------------------------------------------------- @@ -35,7 +56,7 @@ replicaCount: 1 # --------------------------------------------------------------------------- image: registry: ghcr.io - repository: openagrinet/network-adapter + repository: openagrinet/network-adapter # one image for all three roles tag: "latest" digest: "" pullPolicy: IfNotPresent @@ -60,7 +81,7 @@ serviceAccount: # Create it with ExternalSecrets, or by hand from what bin/setup.py wrote into # keys/keys.json in the compose stack: # -# kubectl create secret generic network-adapter-keys \ +# kubectl create secret generic -adapter-keys \ # --from-literal=subscriberId=... \ # --from-literal=keyId=... \ # --from-literal=signingPrivateKey=... \ @@ -111,7 +132,7 @@ configRenderer: # --------------------------------------------------------------------------- # Upstreams # -# In-cluster addresses, and both are read at request time rather than at +# In-cluster address, read at request time rather than at # start-up -- so a registry that is briefly unreachable produces failed # requests, not a crash loop. # --------------------------------------------------------------------------- @@ -122,23 +143,34 @@ registry: entity: Participant providerEntity: ProviderSchema -discovery: - # Where discover and publish are forwarded. No path -- the router appends - # the action, so anything here becomes a prefix on every forwarded call. - url: "" +# --------------------------------------------------------------------------- +# Handler +# +# role: bap originates a call, bpp receives one. Left empty, experience gets bap and +# the other two get bpp. +# steps: left empty, experience gets [addRoute, sign] and the other two get +# [validateSign, addRoute, sign]. experience sits inside the trust boundary and +# accepts unsigned requests, so it has no signature to validate -- that +# difference is the reason this is a value and not a constant. +# --------------------------------------------------------------------------- +handler: + path: / + role: "" + steps: [] # --------------------------------------------------------------------------- # Routing # -# Both actions share one rule because they share a target: publish arrives -# from the provider adapter, discover from the experience adapter, and neither -# is answered here. +# A list, because provider fans out to several upstreams while network and experience +# each have one target. Required: an adapter with no route accepts requests and +# has nowhere to send them. +# +# target.url is a bare host and port with NO path -- targetType "url" appends +# the action, so a trailing slash produces //discover. In-cluster the host is +# the RELEASE name of the target, which is what its Service is called. # --------------------------------------------------------------------------- routing: - version: "2.0.0" - endpoints: - - discover - - publish + rules: [] # --------------------------------------------------------------------------- # Beckn schema validation @@ -179,10 +211,21 @@ otel: # OTLP/gRPC, e.g. "otel-collector.observability.svc.cluster.local:4317" endpoint: "" environment: dev - serviceName: oan-network-adapter + # Left empty, defaults to oan--adapter, matching OTEL_SERVICE_NAME in + # the compose stack so traces from either deployment line up. + serviceName: "" logLevel: info +http: + timeout: + read: 30 + write: 30 + idle: 30 + +# The compose stack uses 9200 for provider, 9201 for network and 9202 for experience. +# Nothing in-cluster requires that -- each adapter has its own Service -- but +# keeping them distinct means a rendered config reads the same in both places. service: type: ClusterIP port: 9201 diff --git a/charts/network-adapter/CHANGELOG.md b/charts/network-adapter/CHANGELOG.md deleted file mode 100644 index b4b9c54..0000000 --- a/charts/network-adapter/CHANGELOG.md +++ /dev/null @@ -1,22 +0,0 @@ -# Changelog - -All notable changes to this chart are documented here. - -The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this chart adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -## [0.1.0] - 2026-09-04 - -### Added - -- Initial chart, modelled on the `network-adapter` service in - `docker-deployment/docker-compose.yml` (#2). -- Config rendered from a ConfigMap of placeholders plus an identity Secret, - substituted by an init container into an `emptyDir` — so the keypair is - never written to a ConfigMap and never appears in a process environment. -- Render-time failures for the five values whose absence would otherwise - produce a pod that runs, reports Ready, and does not work. -- Optional HPA, PodDisruptionBudget, Ingress and upstream readiness gates, all - off by default. diff --git a/charts/network-adapter/ci/lint-values.yaml b/charts/network-adapter/ci/lint-values.yaml deleted file mode 100644 index 6df6795..0000000 --- a/charts/network-adapter/ci/lint-values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Minimum that renders. Every value here is one the chart fails without, which -# makes this file a list of what a real deployment must decide. -image: - registry: ghcr.io - repository: openagrinet/network-adapter - tag: "v1.9.0" - -keys: - existingSecret: - name: network-adapter-keys - -registry: - url: http://registry:8081/api/v1 - -discovery: - url: http://discovery:8080 diff --git a/charts/network-adapter/examples/network-adapter.dev.yaml b/charts/network-adapter/examples/network-adapter.dev.yaml deleted file mode 100644 index be3496f..0000000 --- a/charts/network-adapter/examples/network-adapter.dev.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# network-adapter, dev. -# -# helm upgrade --install network-adapter charts/network-adapter \ -# -n oan -f charts/network-adapter/examples/network-adapter.dev.yaml -# -# The Secret comes first, and it is not optional. From what bin/setup.py wrote -# into keys/keys.json in the compose stack: -# -# kubectl -n oan create secret generic network-adapter-keys \ -# --from-literal=subscriberId="$(jq -r .network.subscriberId keys/keys.json)" \ -# --from-literal=keyId="$(jq -r .network.keyId keys/keys.json)" \ -# --from-literal=signingPrivateKey="$(jq -r .network.signingPrivate keys/keys.json)" \ -# --from-literal=signingPublicKey="$(jq -r .network.signingPublic keys/keys.json)" \ -# --from-literal=encrPrivateKey="$(jq -r .network.encrPrivate keys/keys.json)" \ -# --from-literal=encrPublicKey="$(jq -r .network.encrPublic keys/keys.json)" -# -# Check the field names against your keys.json before running that -- setup.py -# owns that shape, not this chart. - -image: - registry: ghcr.io - repository: openagrinet/network-adapter - tag: "v1.9.0" - # The package is private. Without a docker-registry secret named here the - # deploy looks clean and the pod sits in ImagePullBackOff. - pullSecrets: [] - -keys: - existingSecret: - name: network-adapter-keys - -# In-cluster Services. These are the compose service names as they land in -# Kubernetes -- adjust to whatever your releases are actually called. -registry: - url: http://registry:8081/api/v1 - -discovery: - url: http://discovery:8080 - -logLevel: debug - -# Off until a collector exists. On with no endpoint reachable, the exporter -# logs a failure every interval and buries everything else. -otel: - enabled: false - environment: dev - -replicaCount: 1 - -resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 512Mi diff --git a/charts/network-adapter/templates/NOTES.txt b/charts/network-adapter/templates/NOTES.txt deleted file mode 100644 index 5c5163e..0000000 --- a/charts/network-adapter/templates/NOTES.txt +++ /dev/null @@ -1,41 +0,0 @@ -{{ .Chart.Name }} {{ .Chart.Version }} — {{ include "network-adapter.fullname" . }} - - image {{ include "network-adapter.image" . }} - service {{ include "network-adapter.fullname" . }}:{{ .Values.service.port }} - registry {{ .Values.registry.url }} - discovery {{ .Values.discovery.url }} - telemetry {{ if .Values.otel.enabled }}{{ .Values.otel.endpoint }}{{ else }}off{{ end }} - -Check it is serving: - - kubectl -n {{ include "oan-common.namespace" . }} port-forward svc/{{ include "network-adapter.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} - curl localhost:{{ .Values.service.port }}/health - -/health reports the build actually running, which is worth more than this -chart's appVersion. - -TWO THINGS THIS CHART CANNOT CHECK FOR YOU - -1. The Participant row. This adapter signs as {{ .Values.keys.existingSecret.name }}'s - subscriberId, and every peer verifies that signature against the public key - the registry publishes for it. If the row is missing, or carries a different - key, this pod runs and reports Ready while every peer rejects what it signs - — and the error appears in their logs, not yours. - -2. Key rotation. The keys live in Secret/{{ include "network-adapter.keysSecretName" . }}, - which this chart does not render and therefore cannot include in the config - checksum. Changing that Secret does not restart anything: - - kubectl -n {{ include "oan-common.namespace" . }} rollout restart deploy/{{ include "network-adapter.fullname" . }} - -{{- if not .Values.otel.enabled }} - -Telemetry is off, which is the right setting until a collector exists: with it -on and nothing listening the exporter logs a failure every interval. -{{- end }} -{{- if .Values.ingress.enabled }} - -Ingress is ON. This adapter verifies signatures on the way in, so what keeps -it safe is the registry's contents rather than this route — every caller must -hold a key the registry publishes for them. -{{- end }} diff --git a/charts/network-adapter/templates/_helpers.tpl b/charts/network-adapter/templates/_helpers.tpl deleted file mode 100644 index 6767474..0000000 --- a/charts/network-adapter/templates/_helpers.tpl +++ /dev/null @@ -1,111 +0,0 @@ -{{/* -# ============================================================================ -# NETWORK ADAPTER CHART HELPERS -# Owner: OpenAgriNet Engineering Team -# Purpose: chart-local helpers delegating to oan-common, plus the identity, -# upstream and config-rendering wiring this adapter needs. -# ============================================================================ -*/}} - -{{- define "network-adapter.name" -}} -{{- include "oan-common.name" . -}} -{{- end }} - -{{- define "network-adapter.fullname" -}} -{{- include "oan-common.fullname" . -}} -{{- end }} - -{{- define "network-adapter.labels" -}} -{{- include "oan-common.labels" . -}} -{{- end }} - -{{- define "network-adapter.selectorLabels" -}} -{{- include "oan-common.selectorLabels" . -}} -{{- end }} - -{{- define "network-adapter.serviceAccountName" -}} -{{- include "oan-common.serviceAccount.name" . -}} -{{- end }} - -{{- define "network-adapter.image" -}} -{{- include "oan-common.image" . -}} -{{- end }} - -{{/* -The Secret holding this adapter's keypair. - -Failing here rather than letting the render succeed is the point. Without it -the config would carry the literal placeholders through to the process, which -starts, serves /health, reports Ready, and then fails every signature it -attempts -- at the far end, in a peer's logs, with nothing in this pod saying -why. -*/}} -{{- define "network-adapter.keysSecretName" -}} -{{- $name := .Values.keys.existingSecret.name | default "" -}} -{{- if not $name -}} -{{- fail (printf "%s: keys.existingSecret.name is required. This adapter's identity is a keypair, this chart renders no Secrets, and a pod without one starts and reports Ready while failing every signature it makes. See values.yaml for the six keys it must hold." .Chart.Name) -}} -{{- end -}} -{{- $name -}} -{{- end }} - -{{/* -The registry API base. Required for the same reason as the keys: absent, the -adapter builds a plugin pointed at nothing and every signature verification -fails on a lookup rather than on the signature. -*/}} -{{- define "network-adapter.registryUrl" -}} -{{- $url := .Values.registry.url | default "" -}} -{{- if not $url -}} -{{- fail (printf "%s: registry.url is required -- e.g. http://registry:8081/api/v1. The adapter verifies every caller against the key the registry publishes for them, so with no registry it can verify nobody." .Chart.Name) -}} -{{- end -}} -{{- $url -}} -{{- end }} - -{{/* -Where discover and publish go. No trailing path: targetType "url" appends the -action, so anything here becomes a prefix on every forwarded call. -*/}} -{{- define "network-adapter.discoveryUrl" -}} -{{- $url := .Values.discovery.url | default "" -}} -{{- if not $url -}} -{{- fail (printf "%s: discovery.url is required -- e.g. http://discovery:8080. This adapter forwards discover and publish there and answers neither itself, so with it unset every request it accepts has nowhere to go." .Chart.Name) -}} -{{- end -}} -{{- if hasSuffix "/" $url -}} -{{- fail (printf "%s: discovery.url must not end in a slash (%q). The router appends the action to it, so a trailing slash produces //discover." .Chart.Name $url) -}} -{{- end -}} -{{- $url -}} -{{- end }} - -{{/* -OTLP endpoint, required only when telemetry is on. An enabled exporter with -no endpoint dials nothing and logs a failure per export interval, which is -the noise the three enable flags exist to avoid. -*/}} -{{- define "network-adapter.otlpEndpoint" -}} -{{- if .Values.otel.enabled -}} -{{- $ep := .Values.otel.endpoint | default "" -}} -{{- if not $ep -}} -{{- fail (printf "%s: otel.endpoint is required when otel.enabled is true -- e.g. otel-collector.observability.svc.cluster.local:4317. An enabled exporter with nowhere to send logs a failure every export interval." .Chart.Name) -}} -{{- end -}} -{{- $ep -}} -{{- end -}} -{{- end }} - -{{/* -Where the rendered config lands. An emptyDir, because the init container -writes it and readOnlyRootFilesystem means there is nowhere else to write. -*/}} -{{- define "network-adapter.configDir" -}} -/app/config -{{- end }} - -{{/* -Annotation that rolls the pods when the config changes. - -The keys are not in this checksum and must not be: the Secret is not rendered -by this chart, so its content is not knowable at template time. Rotating a key -therefore needs a rollout restart -- which the NOTES print. -*/}} -{{- define "network-adapter.configChecksum" -}} -{{- include (print $.Template.BasePath "/configmap.yaml") . | sha256sum -}} -{{- end }}