diff --git a/.github/docker-compose.platform.env b/.github/docker-compose.platform.env new file mode 100644 index 0000000..45d2f69 --- /dev/null +++ b/.github/docker-compose.platform.env @@ -0,0 +1,8 @@ +# Single source of truth for the opentdf/platform version pinned in +# docker-compose.platform.override.yaml. Passed to `docker compose` via +# --env-file so PLATFORM_VERSION only has to be declared once. +# +# NanoTDF rewrap (kas ec rewrap) breaks against every platform release +# newer than this that was tried (v0.16.0 through :nightly), so we're +# pinned here for now -- see issue #26. +PLATFORM_VERSION=v0.8.1 diff --git a/.github/docker-compose.platform.override.yaml b/.github/docker-compose.platform.override.yaml new file mode 100644 index 0000000..95822ac --- /dev/null +++ b/.github/docker-compose.platform.override.yaml @@ -0,0 +1,86 @@ +# Overrides for docker-compose.platform.yaml needed to run the platform +# integration tests in CI (and locally). See issue #26 for the background +# on each of these -- briefly: +# +# - The platform's Keycloak provisioning data and policy fixtures must come +# from the SAME git tag as the platform image (docker-compose.platform.yaml +# already pins the image itself to PLATFORM_VERSION, declared once in +# docker-compose.platform.env -- pass that file via `docker compose +# --env-file`). NanoTDF rewrap (kas ec rewrap) breaks against every +# platform release newer than that tag was tried against (v0.16.0 through +# :nightly), independent of otdfctl/SDK version, so everything is pinned +# for now. +# - The Keycloak provisioning data (download-keycloak-data below) is also +# patched with `directAccessGrantsEnabled: true` on the "opentdf" client +# via `yq`, which our integration tests need for the OAuth2 password grant +# (see tests/integration/support_sdk.py::get_user_access_token). This +# mirrors how the pre-docker-compose CI patched the same file with `yq` +# after checking out the platform repo at this tag -- see git history of +# platform-integration-test.yaml before this pipeline used docker compose. +# - Keycloak's plaintext port is published and opentdf.yaml's issuer is +# patched to match, instead of going through Caddy's TLS route. Our +# Connect RPC client for KAS (pyqwest-based) cannot disable TLS +# certificate verification for https:// URLs, so the platform and +# Keycloak endpoints used by the Python SDK/tests need to stay plaintext. +# (otdfctl doesn't have this limitation and is fine going through Caddy.) +# - Every file the base compose file fetches from opentdf/platform@main is +# repinned here to the same git tag as PLATFORM_VERSION (`service/`), +# so nothing in this pipeline tracks main unpinned -- see download-init-script +# below. The one exception is generate-pqc-keys: the PQC keygen tool +# (service/cmd/keygen) doesn't exist yet at service/${PLATFORM_VERSION}, and +# the opentdf.yaml pinned to that tag only references the RSA/EC keys that +# generate-keys already creates, not PQC/hybrid ones -- so its output isn't +# consumed at this pin. It's turned into a no-op below instead of being +# pointed at a ref that doesn't have the tool. Revisit both when +# PLATFORM_VERSION is next bumped. +services: + download-platform-config: + command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/opentdf-example.yaml'] + + download-fixtures: + command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/service/internal/fixtures/policy_fixtures.yaml'] + + download-init-script: + # Unlike download-platform-config/download-fixtures below, the base + # service sets `entrypoint: /bin/sh`, so the command must be shell + # script text (`-c ...`), not an exec-form argv list -- otherwise sh + # tries to run "wget" as a script file and exits 2. + command: + - -c + - | + wget -O /configs/init-temp-keys.sh 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/.github/scripts/init-temp-keys.sh' + + generate-pqc-keys: + image: alpine:3.23 + entrypoint: /bin/sh + command: + - -c + - | + echo "Skipping PQC key generation: service/cmd/keygen doesn't exist at service/${PLATFORM_VERSION}, and its output isn't referenced by opentdf.yaml at that pin." + restart: "no" + + download-keycloak-data: + command: + - -c + - | + apk add --no-cache yq + wget -O /configs/keycloak_data.yaml 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/service/cmd/keycloak_data.yaml' + yq -i '.realms[0].clients[0].client.directAccessGrantsEnabled = true' /configs/keycloak_data.yaml + echo "Downloaded keycloak_data.yaml (service/${PLATFORM_VERSION}) and patched directAccessGrantsEnabled for the opentdf client" + + keycloak: + ports: + - "8888:8888" + + patch-platform-config: + command: + - sh + - -c + - | + apk add --no-cache sed + sed -i 's|http://keycloak:8888|http://keycloak.opentdf.local:8888|g' /configs/opentdf.yaml + echo "Patched opentdf.yaml to use keycloak.opentdf.local:8888 (plaintext)" + +volumes: + configs: + name: opentdf_configs diff --git a/.github/docker-compose.platform.yaml b/.github/docker-compose.platform.yaml new file mode 100644 index 0000000..67d82e2 --- /dev/null +++ b/.github/docker-compose.platform.yaml @@ -0,0 +1,582 @@ +# Vendored from https://opentdf.io/quickstart/docker-compose.yaml +# (the "Getting the Platform Running" section of +# https://opentdf.io/getting-started#getting-the-platform-running), with one +# deliberate deviation from the upstream file: the `platform*` services' +# image tags were changed from the literal `:nightly` to `:${PLATFORM_VERSION}`, +# so the version we're pinned to (see docker-compose.platform.env) is +# declared in exactly one place instead of being overridden after the fact. +# If re-vendoring this file from upstream, reapply that change. +# +# See docker-compose.platform.override.yaml for the other local overrides +# needed to make this usable in CI (see issue #26). +networks: + default: + name: opentdf_platform + +configs: + caddy_config: + content: | + { + log { + level INFO + output stdout + } + } + https://keycloak.opentdf.local:9443 { + tls internal + reverse_proxy keycloak:8888 + } + https://platform.opentdf.local:8443 { + tls internal + reverse_proxy { + to h2c://platform:8080 + transport http { + versions h2c 2 1.1 # Enable gRPC proxying + } + } + } + +services: + caddy: + image: caddy:2.8.4-alpine + command: ['caddy','run', '--config', '/etc/caddy/Caddyfile'] + configs: + - source: caddy_config + target: /etc/caddy/Caddyfile + ports: + - "8443:8443" + - "9443:9443" + - "2019:2019" + volumes: + - caddy_data:/data + depends_on: + ensure-permissions: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:2019/metrics"] + interval: 5s + timeout: 5s + retries: 3 + restart: unless-stopped + + check-certs: + image: alpine:3.23 + volumes: + - type: volume + source: caddy_data + target: /etc/ssl/certs + volume: + subpath: caddy/certificates/local/keycloak.opentdf.local/ + command: + - sh + - -c + - | + echo "Checking certificates" + ls -alh /etc/ssl/certs + cat /etc/ssl/certs/keycloak.opentdf.local.crt + depends_on: + caddy: + condition: service_healthy + ensure-permissions: + condition: service_completed_successfully + restart: "no" + + ensure-permissions: + image: alpine:3.23 + command: + - 'sh' + - '-c' + - | + chmod -R 755 /data + volumes: + - caddy_data:/data + restart: "no" + + keycloak: + volumes: + - keys:/keys:ro + image: ghcr.io/opentdf/keycloak-standard:26.4.0 + restart: always + depends_on: + fix-keys-permissions: + condition: service_completed_successfully + command: + - "start-dev" + - "--verbose" + - "-Djavax.net.ssl.trustStorePassword=password" + - "-Djavax.net.ssl.HostnameVerifier=AllowAll" + - "-Djavax.net.ssl.trustStore=/keys/ca.jks" + - "--spi-truststore-file-hostname-verification-policy=ANY" + environment: + KC_PROXY_HEADERS: xforwarded + KC_HTTP_RELATIVE_PATH: /auth + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_BACKCHANNEL: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: "8888" + KC_HTTPS_PORT: "8443" + KC_HTTP_MANAGEMENT_PORT: "9001" + KC_BOOTSTRAP_ADMIN_USERNAME: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: changeme + #KC_HOSTNAME_URL: http://localhost:8888/auth + KC_FEATURES: "admin-fine-grained-authz:v1" + KC_HEALTH_ENABLED: "true" + KC_HTTPS_KEY_STORE_PASSWORD: "password" + KC_HTTPS_KEY_STORE_FILE: "/keys/ca.jks" + KC_HTTPS_CERTIFICATE_FILE: "/keys/localhost.crt" + KC_HTTPS_CERTIFICATE_KEY_FILE: "/keys/localhost.key" + KC_HTTPS_CLIENT_AUTH: "request" + ### + # The following environment variable resolves SIGILL with Code 134 when running Java processes on Apple M4 chips + # + # On Apple Silicon (M4 chip): + # export JAVA_OPTS_APPEND="-XX:UseSVE=0" + # docker-compose up + # + # On other architectures: + # export JAVA_OPTS_APPEND="" + # docker-compose up + # + # Or set directly: JAVA_OPTS_APPEND="-XX:UseSVE=0" docker-compose up + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + ### + # ports: + # - "${KC_EXPOSE_PORT:-8443}:8443" + # - "${KC_EXPOSE_PORT_HTTP:-8888}:8888" + # - "${KC_EXPOSE_PORT_MGMT:-9001}:9001" + healthcheck: + test: + - CMD-SHELL + - | + [ -f /tmp/HealthCheck.java ] || echo "public class HealthCheck { + public static void main(String[] args) throws java.lang.Throwable { + javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); + javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance(\"SSL\"); + sc.init(null, new javax.net.ssl.TrustManager[]{ + new javax.net.ssl.X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + } + }, new java.security.SecureRandom()); + javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + java.net.HttpURLConnection conn = (java.net.HttpURLConnection)new java.net.URL(args[0]).openConnection(); + System.exit(java.net.HttpURLConnection.HTTP_OK == conn.getResponseCode() ? 0 : 1); + } + }" > /tmp/HealthCheck.java && java ${JAVA_OPTS_APPEND} /tmp/HealthCheck.java http://localhost:8888/auth 2>/dev/null + interval: 10s + timeout: 10s + retries: 10 + start_period: 3m + opentdfdb: + image: postgres:15-alpine + restart: always + user: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme + POSTGRES_DB: opentdf + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "16686:16686" # Web UI + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "14250:14250" # Model/collector gRPC + profiles: + - tracing + restart: always + + # Provision Keycloak with initial configuration + platform-provision-keycloak: + image: registry.opentdf.io/platform:${PLATFORM_VERSION} + command: ["provision", "keycloak", "-e", "https://keycloak.opentdf.local:9443/auth", "-f", "/configs/keycloak_data.yaml"] + depends_on: + keycloak: + condition: service_healthy + opentdfdb: + condition: service_healthy + patch-platform-config: + condition: service_completed_successfully + download-keycloak-data: + condition: service_completed_successfully + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + volumes: + - configs:/configs:ro + - keys:/keys:ro + environment: + - OPENTDF_CONFIG_FILE=/configs/opentdf.yaml + restart: "no" + extra_hosts: + - "keycloak.opentdf.local:host-gateway" + # Prepare fixtures directory structure - create symlink to expected location + prepare-fixtures: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + download-fixtures: + condition: service_completed_successfully + command: + - sh + - -c + - | + mkdir -p /configs/service/internal/fixtures + cd /configs + ln -sf /configs/service/internal/fixtures ./service + restart: "no" + + # Add sample attributes and metadata + platform-provision-fixtures: + image: registry.opentdf.io/platform:${PLATFORM_VERSION} + command: ["provision", "fixtures", "--config-file", "/configs/opentdf.yaml"] + working_dir: /configs + depends_on: + platform-provision-keycloak: + condition: service_completed_successfully + opentdfdb: + condition: service_healthy + prepare-fixtures: + condition: service_completed_successfully + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + volumes: + - configs:/configs:ro + - keys:/keys:ro + restart: "no" + + # Prepare CA certificates bundle with Caddy cert + prepare-ca-certs: + image: alpine:3.23 + volumes: + - type: volume + source: caddy_data + target: /caddy-certs + read_only: true + volume: + subpath: caddy/certificates/local/keycloak.opentdf.local + - platform_certs:/etc/ssl/certs + depends_on: + caddy: + condition: service_healthy + command: + - sh + - -c + - | + # Install ca-certificates package + apk add --no-cache ca-certificates + # Copy Caddy certificate to CA bundle + cp /caddy-certs/keycloak.opentdf.local.crt /usr/local/share/ca-certificates/ + update-ca-certificates + # Copy the updated CA bundle to shared volume + cp -r /etc/ssl/certs/* /etc/ssl/certs/ + echo "CA certificates prepared successfully" + restart: "no" + + # Main OpenTDF Platform server + platform: + image: registry.opentdf.io/platform:${PLATFORM_VERSION} + command: ["start", "--config-file", "/configs/opentdf.yaml"] + depends_on: + platform-provision-fixtures: + condition: service_completed_successfully + keycloak: + condition: service_healthy + opentdfdb: + condition: service_healthy + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + prepare-ca-certs: + condition: service_completed_successfully + ports: + - "8080:8080" + volumes: + - configs:/configs:ro + - keys:/keys:ro + - platform_certs:/etc/ssl/certs:ro + extra_hosts: + - "keycloak.opentdf.local:host-gateway" + restart: unless-stopped + + # Initialize volume permissions + init-volumes: + image: alpine:3.23 + volumes: + - configs:/configs + - keys:/keys + command: + - sh + - -c + - | + chmod 777 /configs /keys + mkdir -p /configs/service/internal/fixtures + chmod -R 777 /configs + restart: "no" + + # Fix keys permissions after generation + fix-keys-permissions: + image: alpine:3.23 + volumes: + - keys:/keys + depends_on: + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + command: + - sh + - -c + - | + chmod -R 755 /keys + chmod 644 /keys/* + restart: "no" + + # Download platform configuration file + download-platform-config: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/main/opentdf-example.yaml'] + restart: "no" + + # Patch platform configuration to use keycloak.opentdf.local:9443 + patch-platform-config: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + download-platform-config: + condition: service_completed_successfully + command: + - sh + - -c + - | + apk add --no-cache sed + sed -i 's|http://keycloak:8888|https://keycloak.opentdf.local:9443|g' /configs/opentdf.yaml + echo "Patched opentdf.yaml to use keycloak.opentdf.local:9443" + restart: "no" + + # Download Keycloak provisioning data + download-keycloak-data: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + URL='https://raw.githubusercontent.com/opentdf/platform/main/service/cmd/keycloak_data.yaml' + OUTPUT='/configs/keycloak_data.yaml' + MAX_ATTEMPTS=3 + + for i in $$(seq 1 $$MAX_ATTEMPTS); do + echo "Attempt $$i of $$MAX_ATTEMPTS: Downloading keycloak_data.yaml..." + + if wget -O "$$OUTPUT" "$$URL"; then + echo "Download successful" + + # Validate the downloaded file + if [ -f "$$OUTPUT" ] && [ -s "$$OUTPUT" ]; then + if head -1 "$$OUTPUT" | grep -q -E '^(---|\w+:)'; then + echo "Validation passed: File exists, non-empty, and appears to be valid YAML" + exit 0 + else + echo "Validation failed: File does not appear to be valid YAML" + rm -f "$$OUTPUT" + fi + else + echo "Validation failed: File is missing or empty" + fi + else + echo "Download failed (attempt $$i)" + fi + + if [ $$i -lt $$MAX_ATTEMPTS ]; then + echo "Retrying in 2 seconds..." + sleep 2 + fi + done + + echo "ERROR: Failed to download and validate keycloak_data.yaml after $$MAX_ATTEMPTS attempts" + exit 1 + restart: "no" + + # Download fixtures data + download-fixtures: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/main/service/internal/fixtures/policy_fixtures.yaml'] + restart: "no" + + # Download init-temp-keys script + download-init-script: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + URL='https://raw.githubusercontent.com/opentdf/platform/main/.github/scripts/init-temp-keys.sh' + OUTPUT='/configs/init-temp-keys.sh' + MAX_ATTEMPTS=3 + + for i in $$(seq 1 $$MAX_ATTEMPTS); do + echo "Attempt $$i of $$MAX_ATTEMPTS: Downloading init-temp-keys.sh..." + + if wget -O "$$OUTPUT" "$$URL"; then + echo "Download successful" + + # Validate the downloaded file + if [ -f "$$OUTPUT" ] && [ -s "$$OUTPUT" ]; then + if head -1 "$$OUTPUT" | grep -q '^#!/'; then + echo "Validation passed: File exists, non-empty, and appears to be a shell script" + exit 0 + else + echo "Validation failed: File does not appear to be a valid shell script" + rm -f "$$OUTPUT" + fi + else + echo "Validation failed: File is missing or empty" + fi + else + echo "Download failed (attempt $$i)" + fi + + if [ $$i -lt $$MAX_ATTEMPTS ]; then + echo "Retrying in 2 seconds..." + sleep 2 + fi + done + + echo "ERROR: Failed to download and validate init-temp-keys.sh after $$MAX_ATTEMPTS attempts" + exit 1 + restart: "no" + + # Generate keys without Docker dependency + generate-keys: + image: alpine:3.23 + volumes: + - configs:/configs + - keys:/keys + depends_on: + download-init-script: + condition: service_completed_successfully + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + apk add --no-cache openssl openjdk11-jre bash + cd /keys + + # Generate KAS RSA private key + openssl genpkey -algorithm RSA -out /keys/kas-private.pem -pkeyopt rsa_keygen_bits:2048 + openssl rsa -in /keys/kas-private.pem -pubout -out /keys/kas-cert.pem + + # Generate ECC Key + openssl ecparam -name prime256v1 > /tmp/ecparams.tmp + openssl req -x509 -nodes -newkey ec:/tmp/ecparams.tmp -subj "/CN=kas" -keyout /keys/kas-ec-private.pem -out /keys/kas-ec-cert.pem -days 365 + + # Generate CA + openssl req -x509 -nodes -newkey RSA:2048 -subj "/CN=ca" -keyout /keys/keycloak-ca-private.pem -out /keys/keycloak-ca.pem -days 365 + + # Generate localhost certificate + printf "subjectAltName=DNS:localhost,IP:127.0.0.1" > /tmp/sanX509.conf + printf "[req]\ndistinguished_name=req_distinguished_name\n[req_distinguished_name]\n[alt_names]\nDNS.1=localhost\nIP.1=127.0.0.1" > /tmp/req.conf + openssl req -new -nodes -newkey rsa:2048 -keyout /keys/localhost.key -out /tmp/localhost.req -batch -subj "/CN=localhost" -config /tmp/req.conf + openssl x509 -req -in /tmp/localhost.req -CA /keys/keycloak-ca.pem -CAkey /keys/keycloak-ca-private.pem -CAcreateserial -out /keys/localhost.crt -days 3650 -sha256 -extfile /tmp/sanX509.conf + + # Generate sample user certificate + openssl req -new -nodes -newkey rsa:2048 -keyout /keys/sampleuser.key -out /tmp/sampleuser.req -batch -subj "/CN=sampleuser" + openssl x509 -req -in /tmp/sampleuser.req -CA /keys/keycloak-ca.pem -CAkey /keys/keycloak-ca-private.pem -CAcreateserial -out /keys/sampleuser.crt -days 3650 + + # Convert to PKCS12 + openssl pkcs12 -export -in /keys/keycloak-ca.pem -inkey /keys/keycloak-ca-private.pem -out /keys/ca.p12 -nodes -passout pass:password + + # Convert PKCS12 to JKS using keytool (no Docker needed) + keytool -importkeystore \ + -srckeystore /keys/ca.p12 \ + -srcstoretype PKCS12 \ + -destkeystore /keys/ca.jks \ + -deststoretype JKS \ + -srcstorepass "password" \ + -deststorepass "password" \ + -noprompt + + echo "Keys generated successfully" + environment: + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + restart: "no" + + # Generate hybrid post-quantum KAS keys (X-Wing, P256+ML-KEM-768, P384+ML-KEM-1024). + # Uses a Go image to build and run the keygen from the platform source. + generate-pqc-keys: + image: golang:1.25-alpine + volumes: + - keys:/keys + depends_on: + generate-keys: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + set -e + apk add --no-cache git + WORKDIR=$$(mktemp -d) + cd "$$WORKDIR" + git init -q + git remote add origin https://github.com/opentdf/platform.git + git config core.sparseCheckout true + echo "lib/" >> .git/info/sparse-checkout + echo "service/cmd/keygen/" >> .git/info/sparse-checkout + echo "service/go.mod" >> .git/info/sparse-checkout + echo "service/go.sum" >> .git/info/sparse-checkout + echo "protocol/" >> .git/info/sparse-checkout + echo "sdk/" >> .git/info/sparse-checkout + git pull --depth 1 -q origin main + cd service + GOWORK=off go run ./cmd/keygen -output /keys + echo "PQC keys generated successfully" + rm -rf "$$WORKDIR" + restart: "no" + +volumes: + keys: + name: opentdf_keys + configs: + name: opentdf_configs + caddy_data: + platform_certs: + name: opentdf_platform_certs diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a3216e6..868a129 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -1,8 +1,11 @@ -# Based on -# https://github.com/opentdf/java-sdk/blob/v0.6.1/.github/workflows/checks.yaml +# Runs the OpenTDF platform locally via `docker compose`, following +# https://opentdf.io/getting-started#getting-the-platform-running +# instead of building the platform from source (see issue #26). # -# Except, that this is a "Composite Action", and specifies 'shell: bash' for -# each 'run:' step. +# docker-compose.platform.yaml is a vendored copy of the quickstart's +# docker-compose.yaml. docker-compose.platform.override.yaml carries the +# local changes needed to make it usable here -- see the comments in that +# file for why each one is needed. name: "NEW: Platform Integration testing" on: @@ -25,111 +28,60 @@ jobs: - name: Checkout this repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - # - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Update /etc/hosts + shell: bash + run: | + echo -e "127.0.0.1 platform.opentdf.local\n127.0.0.1 keycloak.opentdf.local" | sudo tee -a /etc/hosts + + - name: Bring the platform up + shell: bash + run: | + # Note: intentionally not using `--wait` here. This compose file + # has many one-shot init/provisioning containers (restart: "no"), + # and `docker compose up --wait` unreliably reports failure for + # those even when every container has exited cleanly -- readiness + # is instead confirmed below by polling grpcurl. + docker compose \ + --file .github/docker-compose.platform.yaml \ + --file .github/docker-compose.platform.override.yaml \ + --env-file .github/docker-compose.platform.env \ + up -d - - name: Check out platform - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - repository: opentdf/platform - ref: service/v0.8.2 - path: platform - name: Set up go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e with: go-version: "1.24.x" check-latest: false - cache-dependency-path: | - platform/service/go.sum - platform/examples/go.sum - platform/protocol/go/go.sum - platform/sdk/go.sum - - run: go mod download - shell: bash - working-directory: platform - - run: go mod verify - shell: bash - working-directory: platform - - name: Create keys - shell: bash - run: | - .github/scripts/init-temp-keys.sh - # Edit Keycloak sample file for our use case - yq -i '.realms[0].clients[0].client.directAccessGrantsEnabled = true | .realms[0].clients[0].client.serviceAccountsEnabled = true' service/cmd/keycloak_data.yaml - yq -i '.realms[0].clients[1].client.directAccessGrantsEnabled = true | .realms[0].clients[1].client.serviceAccountsEnabled = true' service/cmd/keycloak_data.yaml - yq -i '.realms[0].clients[4].client.directAccessGrantsEnabled = true | .realms[0].clients[4].client.serviceAccountsEnabled = true' service/cmd/keycloak_data.yaml - - cp opentdf-example.yaml opentdf.yaml - # Edit 'opentdf.yaml' for our use case - yq -i 'del(.db) | .services.entityresolution.url = "http://localhost:8888/auth" | .server.auth.issuer = "http://localhost:8888/auth/realms/opentdf"' opentdf.yaml - # The above expression can also be written as 3 separate commands: - # yq -i 'del(.db)' opentdf.yaml - # yq -i '.services.entityresolution.url = "http://localhost:8888/auth"' opentdf.yaml - # yq -i '.server.auth.issuer = "http://localhost:8888/auth/realms/opentdf"' opentdf.yaml - yq -i ' - .server.cryptoProvider = { - "type": "standard", - "standard": { - "keys": [ - { - "kid": "r1", - "alg": "rsa:2048", - "private": "kas-private.pem", - "cert": "kas-cert.pem" - }, - { - "kid": "e1", - "alg": "ec:secp256r1", - "private": "kas-ec-private.pem", - "cert": "kas-ec-cert.pem" - } - ] - } - } - ' opentdf.yaml - sudo chmod -R 777 ./keys - working-directory: platform - # - name: Trust the locally issued cert - # run: | - # keytool \ - # -importcert \ - # -storepass changeit \ - # -noprompt \ - # -file localhost.crt \ - # -keystore $JAVA_HOME/lib/security/cacerts \ - # -alias localhost-for-tests - # working-directory: platform/keys - - name: Bring the services up - shell: bash - run: docker compose up -d --wait --wait-timeout 240 - working-directory: platform - - name: Provision keycloak - shell: bash - run: go run ./service provision keycloak - working-directory: platform - - name: Provision fixtures - shell: bash - run: go run ./service provision fixtures - working-directory: platform - - name: Start server in background - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 - with: - run: | - go run ./service start - wait-on: | - tcp:localhost:8080 - log-output-if: true - wait-for: 90s - working-directory: platform + - name: Get grpcurl shell: bash run: go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.8.9 + - name: Make sure that the platform is up shell: bash run: | - grpcurl -plaintext localhost:8080 list && \ - grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + export PATH=$PATH:$(go env GOPATH)/bin + for i in $(seq 1 60); do + if grpcurl -plaintext localhost:8080 list >/dev/null 2>&1; then + echo "Platform is up after ${i} attempt(s)" + grpcurl -plaintext localhost:8080 list + grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + exit 0 + fi + sleep 5 + done + echo "::error::Platform did not come up in time" + docker compose \ + --file .github/docker-compose.platform.yaml \ + --file .github/docker-compose.platform.override.yaml \ + --env-file .github/docker-compose.platform.env \ + ps -a + docker compose \ + --file .github/docker-compose.platform.yaml \ + --file .github/docker-compose.platform.override.yaml \ + --env-file .github/docker-compose.platform.env \ + logs platform + exit 1 - name: Install otdfctl run: go install github.com/opentdf/otdfctl@v0.28.0 @@ -146,7 +98,7 @@ jobs: - name: Encrypt file with otdfctl (no attributes) run: | export PATH=$PATH:$(go env GOPATH)/bin - otdfctl encrypt -o secret.txt.tdf --host http://localhost:8080 --tls-no-verify --with-client-creds-file creds.json secret.txt + otdfctl encrypt -o secret.txt.tdf --host http://localhost:8080 --with-client-creds-file creds.json secret.txt shell: bash - name: Set up uv @@ -161,7 +113,7 @@ jobs: OPENTDF_CLIENT_ID: "opentdf" OPENTDF_CLIENT_SECRET: "secret" OPENTDF_HOSTNAME: "localhost:8080" - OIDC_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token" + OIDC_TOKEN_ENDPOINT: "http://keycloak.opentdf.local:8888/auth/realms/opentdf/protocol/openid-connect/token" OPENTDF_KAS_URL: "http://localhost:8080/kas" INSECURE_SKIP_VERIFY: "TRUE" TEST_OPENTDF_ATTRIBUTE_1: "https://example.net/attr/attr1/value/value1" @@ -178,7 +130,8 @@ jobs: OPENTDF_CLIENT_SECRET: "secret" OPENTDF_PLATFORM_HOST: "localhost:8080" OPENTDF_PLATFORM_URL: "http://localhost:8080" - OIDC_OP_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token" + OPENTDF_KEYCLOAK_HOST: "keycloak.opentdf.local:8888" + OIDC_OP_TOKEN_ENDPOINT: "http://keycloak.opentdf.local:8888/auth/realms/opentdf/protocol/openid-connect/token" OPENTDF_KAS_URL: "http://localhost:8080/kas" INSECURE_SKIP_VERIFY: "TRUE" TEST_OPENTDF_ATTRIBUTE_1: "https://example.net/attr/attr1/value/value1" diff --git a/packages/otdf-python/src/otdf_python/sdk_builder.py b/packages/otdf-python/src/otdf_python/sdk_builder.py index c320c2d..a02500d 100644 --- a/packages/otdf-python/src/otdf_python/sdk_builder.py +++ b/packages/otdf-python/src/otdf_python/sdk_builder.py @@ -273,7 +273,9 @@ def _discover_token_endpoint_from_platform(self) -> None: ) config_doc = response.json() - configuration = config_doc.get("configuration", {}) + # Newer platform versions return the fields directly at the top + # level instead of nested under a "configuration" key. + configuration = config_doc.get("configuration") or config_doc # Try to get token endpoint from IDP configuration idp_config = configuration.get("idp", {})