From 014772f4be53f8cc261e288d4336e5efe7cbcf77 Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 2 Aug 2026 19:49:43 -0400 Subject: [PATCH 1/7] Run platform integration tests via docker compose Replaces Go source build with quickstart compose --- .github/docker-compose.platform.override.yaml | 60 ++ .github/docker-compose.platform.yaml | 577 ++++++++++++++++++ .github/keycloak_data.yaml | 123 ++++ .../workflows/platform-integration-test.yaml | 124 +--- 4 files changed, 786 insertions(+), 98 deletions(-) create mode 100644 .github/docker-compose.platform.override.yaml create mode 100644 .github/docker-compose.platform.yaml create mode 100644 .github/keycloak_data.yaml diff --git a/.github/docker-compose.platform.override.yaml b/.github/docker-compose.platform.override.yaml new file mode 100644 index 0000000..fc9b017 --- /dev/null +++ b/.github/docker-compose.platform.override.yaml @@ -0,0 +1,60 @@ +# Overrides for docker-compose.platform.yaml needed to run the platform +# integration tests in CI (and locally). See issue #26 for the background +# on each of these -- briefly: +# +# - The platform, its Keycloak provisioning data, and its policy fixtures +# must all come from the SAME git tag. NanoTDF rewrap (kas ec rewrap) +# breaks against every platform release newer than service/v0.8.1 that +# was tried (v0.16.0 through :nightly), independent of otdfctl/SDK +# version, so everything is pinned to that tag for now. +# - Our own keycloak_data.yaml (matching the v0.8.1 realm schema) replaces +# the copy the base compose file downloads from opentdf/platform@main, +# since we need `directAccessGrantsEnabled: true` on the "opentdf" client +# for the OAuth2 password grant used by the PE integration tests. +# - Keycloak's plaintext port is published and opentdf.yaml's issuer is +# patched to match, instead of going through Caddy's TLS route. Our +# Connect RPC client for KAS (pyqwest-based) cannot disable TLS +# certificate verification for https:// URLs, so the platform and +# Keycloak endpoints used by the Python SDK/tests need to stay plaintext. +# (otdfctl doesn't have this limitation and is fine going through Caddy.) +services: + platform-provision-keycloak: + image: registry.opentdf.io/platform:v0.8.1 + platform-provision-fixtures: + image: registry.opentdf.io/platform:v0.8.1 + platform: + image: registry.opentdf.io/platform:v0.8.1 + + download-platform-config: + command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/v0.8.1/opentdf-example.yaml'] + + download-fixtures: + command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/v0.8.1/service/internal/fixtures/policy_fixtures.yaml'] + + download-keycloak-data: + entrypoint: /bin/sh + command: + - -c + - | + cp /custom/keycloak_data.yaml /configs/keycloak_data.yaml + echo "Copied local keycloak_data.yaml into /configs" + volumes: + - configs:/configs + - ./keycloak_data.yaml:/custom/keycloak_data.yaml:ro + + keycloak: + ports: + - "8888:8888" + + patch-platform-config: + command: + - sh + - -c + - | + apk add --no-cache sed + sed -i 's|http://keycloak:8888|http://keycloak.opentdf.local:8888|g' /configs/opentdf.yaml + echo "Patched opentdf.yaml to use keycloak.opentdf.local:8888 (plaintext)" + +volumes: + configs: + name: opentdf_configs diff --git a/.github/docker-compose.platform.yaml b/.github/docker-compose.platform.yaml new file mode 100644 index 0000000..be3a42a --- /dev/null +++ b/.github/docker-compose.platform.yaml @@ -0,0 +1,577 @@ +# Vendored from https://opentdf.io/quickstart/docker-compose.yaml +# (the "Getting the Platform Running" section of +# https://opentdf.io/getting-started#getting-the-platform-running). +# +# See docker-compose.platform.override.yaml for the local overrides needed +# to make this usable in CI (see issue #26). +networks: + default: + name: opentdf_platform + +configs: + caddy_config: + content: | + { + log { + level INFO + output stdout + } + } + https://keycloak.opentdf.local:9443 { + tls internal + reverse_proxy keycloak:8888 + } + https://platform.opentdf.local:8443 { + tls internal + reverse_proxy { + to h2c://platform:8080 + transport http { + versions h2c 2 1.1 # Enable gRPC proxying + } + } + } + +services: + caddy: + image: caddy:2.8.4-alpine + command: ['caddy','run', '--config', '/etc/caddy/Caddyfile'] + configs: + - source: caddy_config + target: /etc/caddy/Caddyfile + ports: + - "8443:8443" + - "9443:9443" + - "2019:2019" + volumes: + - caddy_data:/data + depends_on: + ensure-permissions: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:2019/metrics"] + interval: 5s + timeout: 5s + retries: 3 + restart: unless-stopped + + check-certs: + image: alpine:3.23 + volumes: + - type: volume + source: caddy_data + target: /etc/ssl/certs + volume: + subpath: caddy/certificates/local/keycloak.opentdf.local/ + command: + - sh + - -c + - | + echo "Checking certificates" + ls -alh /etc/ssl/certs + cat /etc/ssl/certs/keycloak.opentdf.local.crt + depends_on: + caddy: + condition: service_healthy + ensure-permissions: + condition: service_completed_successfully + restart: "no" + + ensure-permissions: + image: alpine:3.23 + command: + - 'sh' + - '-c' + - | + chmod -R 755 /data + volumes: + - caddy_data:/data + restart: "no" + + keycloak: + volumes: + - keys:/keys:ro + image: ghcr.io/opentdf/keycloak-standard:26.4.0 + restart: always + depends_on: + fix-keys-permissions: + condition: service_completed_successfully + command: + - "start-dev" + - "--verbose" + - "-Djavax.net.ssl.trustStorePassword=password" + - "-Djavax.net.ssl.HostnameVerifier=AllowAll" + - "-Djavax.net.ssl.trustStore=/keys/ca.jks" + - "--spi-truststore-file-hostname-verification-policy=ANY" + environment: + KC_PROXY_HEADERS: xforwarded + KC_HTTP_RELATIVE_PATH: /auth + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_BACKCHANNEL: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: "8888" + KC_HTTPS_PORT: "8443" + KC_HTTP_MANAGEMENT_PORT: "9001" + KC_BOOTSTRAP_ADMIN_USERNAME: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: changeme + #KC_HOSTNAME_URL: http://localhost:8888/auth + KC_FEATURES: "admin-fine-grained-authz:v1" + KC_HEALTH_ENABLED: "true" + KC_HTTPS_KEY_STORE_PASSWORD: "password" + KC_HTTPS_KEY_STORE_FILE: "/keys/ca.jks" + KC_HTTPS_CERTIFICATE_FILE: "/keys/localhost.crt" + KC_HTTPS_CERTIFICATE_KEY_FILE: "/keys/localhost.key" + KC_HTTPS_CLIENT_AUTH: "request" + ### + # The following environment variable resolves SIGILL with Code 134 when running Java processes on Apple M4 chips + # + # On Apple Silicon (M4 chip): + # export JAVA_OPTS_APPEND="-XX:UseSVE=0" + # docker-compose up + # + # On other architectures: + # export JAVA_OPTS_APPEND="" + # docker-compose up + # + # Or set directly: JAVA_OPTS_APPEND="-XX:UseSVE=0" docker-compose up + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + ### + # ports: + # - "${KC_EXPOSE_PORT:-8443}:8443" + # - "${KC_EXPOSE_PORT_HTTP:-8888}:8888" + # - "${KC_EXPOSE_PORT_MGMT:-9001}:9001" + healthcheck: + test: + - CMD-SHELL + - | + [ -f /tmp/HealthCheck.java ] || echo "public class HealthCheck { + public static void main(String[] args) throws java.lang.Throwable { + javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); + javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance(\"SSL\"); + sc.init(null, new javax.net.ssl.TrustManager[]{ + new javax.net.ssl.X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + } + }, new java.security.SecureRandom()); + javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + java.net.HttpURLConnection conn = (java.net.HttpURLConnection)new java.net.URL(args[0]).openConnection(); + System.exit(java.net.HttpURLConnection.HTTP_OK == conn.getResponseCode() ? 0 : 1); + } + }" > /tmp/HealthCheck.java && java ${JAVA_OPTS_APPEND} /tmp/HealthCheck.java http://localhost:8888/auth 2>/dev/null + interval: 10s + timeout: 10s + retries: 10 + start_period: 3m + opentdfdb: + image: postgres:15-alpine + restart: always + user: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme + POSTGRES_DB: opentdf + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "16686:16686" # Web UI + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "14250:14250" # Model/collector gRPC + profiles: + - tracing + restart: always + + # Provision Keycloak with initial configuration + platform-provision-keycloak: + image: registry.opentdf.io/platform:nightly + command: ["provision", "keycloak", "-e", "https://keycloak.opentdf.local:9443/auth", "-f", "/configs/keycloak_data.yaml"] + depends_on: + keycloak: + condition: service_healthy + opentdfdb: + condition: service_healthy + patch-platform-config: + condition: service_completed_successfully + download-keycloak-data: + condition: service_completed_successfully + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + volumes: + - configs:/configs:ro + - keys:/keys:ro + environment: + - OPENTDF_CONFIG_FILE=/configs/opentdf.yaml + restart: "no" + extra_hosts: + - "keycloak.opentdf.local:host-gateway" + # Prepare fixtures directory structure - create symlink to expected location + prepare-fixtures: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + download-fixtures: + condition: service_completed_successfully + command: + - sh + - -c + - | + mkdir -p /configs/service/internal/fixtures + cd /configs + ln -sf /configs/service/internal/fixtures ./service + restart: "no" + + # Add sample attributes and metadata + platform-provision-fixtures: + image: registry.opentdf.io/platform:nightly + command: ["provision", "fixtures", "--config-file", "/configs/opentdf.yaml"] + working_dir: /configs + depends_on: + platform-provision-keycloak: + condition: service_completed_successfully + opentdfdb: + condition: service_healthy + prepare-fixtures: + condition: service_completed_successfully + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + volumes: + - configs:/configs:ro + - keys:/keys:ro + restart: "no" + + # Prepare CA certificates bundle with Caddy cert + prepare-ca-certs: + image: alpine:3.23 + volumes: + - type: volume + source: caddy_data + target: /caddy-certs + read_only: true + volume: + subpath: caddy/certificates/local/keycloak.opentdf.local + - platform_certs:/etc/ssl/certs + depends_on: + caddy: + condition: service_healthy + command: + - sh + - -c + - | + # Install ca-certificates package + apk add --no-cache ca-certificates + # Copy Caddy certificate to CA bundle + cp /caddy-certs/keycloak.opentdf.local.crt /usr/local/share/ca-certificates/ + update-ca-certificates + # Copy the updated CA bundle to shared volume + cp -r /etc/ssl/certs/* /etc/ssl/certs/ + echo "CA certificates prepared successfully" + restart: "no" + + # Main OpenTDF Platform server + platform: + image: registry.opentdf.io/platform:nightly + command: ["start", "--config-file", "/configs/opentdf.yaml"] + depends_on: + platform-provision-fixtures: + condition: service_completed_successfully + keycloak: + condition: service_healthy + opentdfdb: + condition: service_healthy + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + prepare-ca-certs: + condition: service_completed_successfully + ports: + - "8080:8080" + volumes: + - configs:/configs:ro + - keys:/keys:ro + - platform_certs:/etc/ssl/certs:ro + extra_hosts: + - "keycloak.opentdf.local:host-gateway" + restart: unless-stopped + + # Initialize volume permissions + init-volumes: + image: alpine:3.23 + volumes: + - configs:/configs + - keys:/keys + command: + - sh + - -c + - | + chmod 777 /configs /keys + mkdir -p /configs/service/internal/fixtures + chmod -R 777 /configs + restart: "no" + + # Fix keys permissions after generation + fix-keys-permissions: + image: alpine:3.23 + volumes: + - keys:/keys + depends_on: + generate-keys: + condition: service_completed_successfully + generate-pqc-keys: + condition: service_completed_successfully + command: + - sh + - -c + - | + chmod -R 755 /keys + chmod 644 /keys/* + restart: "no" + + # Download platform configuration file + download-platform-config: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/main/opentdf-example.yaml'] + restart: "no" + + # Patch platform configuration to use keycloak.opentdf.local:9443 + patch-platform-config: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + download-platform-config: + condition: service_completed_successfully + command: + - sh + - -c + - | + apk add --no-cache sed + sed -i 's|http://keycloak:8888|https://keycloak.opentdf.local:9443|g' /configs/opentdf.yaml + echo "Patched opentdf.yaml to use keycloak.opentdf.local:9443" + restart: "no" + + # Download Keycloak provisioning data + download-keycloak-data: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + URL='https://raw.githubusercontent.com/opentdf/platform/main/service/cmd/keycloak_data.yaml' + OUTPUT='/configs/keycloak_data.yaml' + MAX_ATTEMPTS=3 + + for i in $$(seq 1 $$MAX_ATTEMPTS); do + echo "Attempt $$i of $$MAX_ATTEMPTS: Downloading keycloak_data.yaml..." + + if wget -O "$$OUTPUT" "$$URL"; then + echo "Download successful" + + # Validate the downloaded file + if [ -f "$$OUTPUT" ] && [ -s "$$OUTPUT" ]; then + if head -1 "$$OUTPUT" | grep -q -E '^(---|\w+:)'; then + echo "Validation passed: File exists, non-empty, and appears to be valid YAML" + exit 0 + else + echo "Validation failed: File does not appear to be valid YAML" + rm -f "$$OUTPUT" + fi + else + echo "Validation failed: File is missing or empty" + fi + else + echo "Download failed (attempt $$i)" + fi + + if [ $$i -lt $$MAX_ATTEMPTS ]; then + echo "Retrying in 2 seconds..." + sleep 2 + fi + done + + echo "ERROR: Failed to download and validate keycloak_data.yaml after $$MAX_ATTEMPTS attempts" + exit 1 + restart: "no" + + # Download fixtures data + download-fixtures: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/main/service/internal/fixtures/policy_fixtures.yaml'] + restart: "no" + + # Download init-temp-keys script + download-init-script: + image: alpine:3.23 + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + URL='https://raw.githubusercontent.com/opentdf/platform/main/.github/scripts/init-temp-keys.sh' + OUTPUT='/configs/init-temp-keys.sh' + MAX_ATTEMPTS=3 + + for i in $$(seq 1 $$MAX_ATTEMPTS); do + echo "Attempt $$i of $$MAX_ATTEMPTS: Downloading init-temp-keys.sh..." + + if wget -O "$$OUTPUT" "$$URL"; then + echo "Download successful" + + # Validate the downloaded file + if [ -f "$$OUTPUT" ] && [ -s "$$OUTPUT" ]; then + if head -1 "$$OUTPUT" | grep -q '^#!/'; then + echo "Validation passed: File exists, non-empty, and appears to be a shell script" + exit 0 + else + echo "Validation failed: File does not appear to be a valid shell script" + rm -f "$$OUTPUT" + fi + else + echo "Validation failed: File is missing or empty" + fi + else + echo "Download failed (attempt $$i)" + fi + + if [ $$i -lt $$MAX_ATTEMPTS ]; then + echo "Retrying in 2 seconds..." + sleep 2 + fi + done + + echo "ERROR: Failed to download and validate init-temp-keys.sh after $$MAX_ATTEMPTS attempts" + exit 1 + restart: "no" + + # Generate keys without Docker dependency + generate-keys: + image: alpine:3.23 + volumes: + - configs:/configs + - keys:/keys + depends_on: + download-init-script: + condition: service_completed_successfully + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + apk add --no-cache openssl openjdk11-jre bash + cd /keys + + # Generate KAS RSA private key + openssl genpkey -algorithm RSA -out /keys/kas-private.pem -pkeyopt rsa_keygen_bits:2048 + openssl rsa -in /keys/kas-private.pem -pubout -out /keys/kas-cert.pem + + # Generate ECC Key + openssl ecparam -name prime256v1 > /tmp/ecparams.tmp + openssl req -x509 -nodes -newkey ec:/tmp/ecparams.tmp -subj "/CN=kas" -keyout /keys/kas-ec-private.pem -out /keys/kas-ec-cert.pem -days 365 + + # Generate CA + openssl req -x509 -nodes -newkey RSA:2048 -subj "/CN=ca" -keyout /keys/keycloak-ca-private.pem -out /keys/keycloak-ca.pem -days 365 + + # Generate localhost certificate + printf "subjectAltName=DNS:localhost,IP:127.0.0.1" > /tmp/sanX509.conf + printf "[req]\ndistinguished_name=req_distinguished_name\n[req_distinguished_name]\n[alt_names]\nDNS.1=localhost\nIP.1=127.0.0.1" > /tmp/req.conf + openssl req -new -nodes -newkey rsa:2048 -keyout /keys/localhost.key -out /tmp/localhost.req -batch -subj "/CN=localhost" -config /tmp/req.conf + openssl x509 -req -in /tmp/localhost.req -CA /keys/keycloak-ca.pem -CAkey /keys/keycloak-ca-private.pem -CAcreateserial -out /keys/localhost.crt -days 3650 -sha256 -extfile /tmp/sanX509.conf + + # Generate sample user certificate + openssl req -new -nodes -newkey rsa:2048 -keyout /keys/sampleuser.key -out /tmp/sampleuser.req -batch -subj "/CN=sampleuser" + openssl x509 -req -in /tmp/sampleuser.req -CA /keys/keycloak-ca.pem -CAkey /keys/keycloak-ca-private.pem -CAcreateserial -out /keys/sampleuser.crt -days 3650 + + # Convert to PKCS12 + openssl pkcs12 -export -in /keys/keycloak-ca.pem -inkey /keys/keycloak-ca-private.pem -out /keys/ca.p12 -nodes -passout pass:password + + # Convert PKCS12 to JKS using keytool (no Docker needed) + keytool -importkeystore \ + -srckeystore /keys/ca.p12 \ + -srcstoretype PKCS12 \ + -destkeystore /keys/ca.jks \ + -deststoretype JKS \ + -srcstorepass "password" \ + -deststorepass "password" \ + -noprompt + + echo "Keys generated successfully" + environment: + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + restart: "no" + + # Generate hybrid post-quantum KAS keys (X-Wing, P256+ML-KEM-768, P384+ML-KEM-1024). + # Uses a Go image to build and run the keygen from the platform source. + generate-pqc-keys: + image: golang:1.25-alpine + volumes: + - keys:/keys + depends_on: + generate-keys: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + set -e + apk add --no-cache git + WORKDIR=$$(mktemp -d) + cd "$$WORKDIR" + git init -q + git remote add origin https://github.com/opentdf/platform.git + git config core.sparseCheckout true + echo "lib/" >> .git/info/sparse-checkout + echo "service/cmd/keygen/" >> .git/info/sparse-checkout + echo "service/go.mod" >> .git/info/sparse-checkout + echo "service/go.sum" >> .git/info/sparse-checkout + echo "protocol/" >> .git/info/sparse-checkout + echo "sdk/" >> .git/info/sparse-checkout + git pull --depth 1 -q origin main + cd service + GOWORK=off go run ./cmd/keygen -output /keys + echo "PQC keys generated successfully" + rm -rf "$$WORKDIR" + restart: "no" + +volumes: + keys: + name: opentdf_keys + configs: + name: opentdf_configs + caddy_data: + platform_certs: + name: opentdf_platform_certs diff --git a/.github/keycloak_data.yaml b/.github/keycloak_data.yaml new file mode 100644 index 0000000..01c187a --- /dev/null +++ b/.github/keycloak_data.yaml @@ -0,0 +1,123 @@ +# Vendored from +# https://raw.githubusercontent.com/opentdf/platform/service/v0.8.1/service/cmd/keycloak_data.yaml +# with one change: `directAccessGrantsEnabled: true` added to the "opentdf" +# client, which our integration tests need for the OAuth2 password grant +# (see tests/integration/support_sdk.py::get_user_access_token). +# +# This must stay pinned to the same platform version/tag as the images in +# docker-compose.platform.override.yaml -- the realm schema this file uses +# is not compatible with newer platform releases (see issue #26). +baseUrl: &baseUrl http://localhost:8888 +serverBaseUrl: &serverBaseUrl http://localhost:8080 +customAudMapper: &customAudMapper + name: audience-mapper + protocol: openid-connect + protocolMapper: oidc-audience-mapper + config: + included.custom.audience: *serverBaseUrl + access.token.claim: "true" + id.token.claim: "true" +realms: + - realm_repepresentation: + realm: opentdf + enabled: true + custom_realm_roles: + - name: opentdf-admin + - name: opentdf-standard + custom_client_roles: + tdf-entity-resolution: + - name: entity-resolution-test-role + custom_groups: + - name: mygroup + attributes: + mygroupattribute: + - mygroupvalue + clients: + - client: + clientID: opentdf + enabled: true + name: opentdf + serviceAccountsEnabled: true + clientAuthenticatorType: client-secret + secret: secret + protocolMappers: + - *customAudMapper + directAccessGrantsEnabled: true + sa_realm_roles: + - opentdf-admin + copies: 10 + - client: + clientID: opentdf-sdk + enabled: true + name: opentdf-sdk + serviceAccountsEnabled: true + clientAuthenticatorType: client-secret + secret: secret + protocolMappers: + - *customAudMapper + sa_realm_roles: + - opentdf-standard + - client: + clientID: tdf-entity-resolution + enabled: true + name: tdf-entity-resolution + serviceAccountsEnabled: true + clientAuthenticatorType: client-secret + secret: secret + protocolMappers: + - *customAudMapper + sa_client_roles: + realm-management: + - view-clients + - query-clients + - view-users + - query-users + - client: + clientID: tdf-authorization-svc + enabled: true + name: tdf-authorization-svc + serviceAccountsEnabled: true + clientAuthenticatorType: client-secret + secret: secret + protocolMappers: + - *customAudMapper + - client: + clientID: cli-client + enabled: true + name: cli-client + serviceAccountsEnabled: false + publicClient: true + redirectUris: + - 'http://localhost:*' + protocolMappers: + - *customAudMapper + users: + - username: sample-user + enabled: true + firstName: sample + lastName: user + email: sampleuser@sample.com + credentials: + - value: testuser123 + type: password + attributes: + superhero_name: + - thor + superhero_group: + - avengers + groups: + - mygroup + realmRoles: + - opentdf-admin + clientRoles: + realm-management: + - view-clients + - query-clients + - view-users + - query-users + tdf-entity-resolution: + - entity-resolution-test-role + copies: 10 + token_exchanges: + - start_client: opentdf + target_client: opentdf-sdk diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index a3216e6..1920801 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -1,8 +1,11 @@ -# Based on -# https://github.com/opentdf/java-sdk/blob/v0.6.1/.github/workflows/checks.yaml +# Runs the OpenTDF platform locally via `docker compose`, following +# https://opentdf.io/getting-started#getting-the-platform-running +# instead of building the platform from source (see issue #26). # -# Except, that this is a "Composite Action", and specifies 'shell: bash' for -# each 'run:' step. +# docker-compose.platform.yaml is a vendored copy of the quickstart's +# docker-compose.yaml. docker-compose.platform.override.yaml and +# keycloak_data.yaml carry the local changes needed to make it usable here +# -- see the comments in those files for why each one is needed. name: "NEW: Platform Integration testing" on: @@ -25,109 +28,33 @@ jobs: - name: Checkout this repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - # - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Update /etc/hosts + shell: bash + run: | + echo -e "127.0.0.1 platform.opentdf.local\n127.0.0.1 keycloak.opentdf.local" | sudo tee -a /etc/hosts + + - name: Bring the platform up + shell: bash + run: | + docker compose \ + --file .github/docker-compose.platform.yaml \ + --file .github/docker-compose.platform.override.yaml \ + up -d --wait --wait-timeout 360 - - name: Check out platform - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - repository: opentdf/platform - ref: service/v0.8.2 - path: platform - name: Set up go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e with: go-version: "1.24.x" check-latest: false - cache-dependency-path: | - platform/service/go.sum - platform/examples/go.sum - platform/protocol/go/go.sum - platform/sdk/go.sum - - run: go mod download - shell: bash - working-directory: platform - - run: go mod verify - shell: bash - working-directory: platform - - name: Create keys - shell: bash - run: | - .github/scripts/init-temp-keys.sh - # Edit Keycloak sample file for our use case - yq -i '.realms[0].clients[0].client.directAccessGrantsEnabled = true | .realms[0].clients[0].client.serviceAccountsEnabled = true' service/cmd/keycloak_data.yaml - yq -i '.realms[0].clients[1].client.directAccessGrantsEnabled = true | .realms[0].clients[1].client.serviceAccountsEnabled = true' service/cmd/keycloak_data.yaml - yq -i '.realms[0].clients[4].client.directAccessGrantsEnabled = true | .realms[0].clients[4].client.serviceAccountsEnabled = true' service/cmd/keycloak_data.yaml - - cp opentdf-example.yaml opentdf.yaml - # Edit 'opentdf.yaml' for our use case - yq -i 'del(.db) | .services.entityresolution.url = "http://localhost:8888/auth" | .server.auth.issuer = "http://localhost:8888/auth/realms/opentdf"' opentdf.yaml - # The above expression can also be written as 3 separate commands: - # yq -i 'del(.db)' opentdf.yaml - # yq -i '.services.entityresolution.url = "http://localhost:8888/auth"' opentdf.yaml - # yq -i '.server.auth.issuer = "http://localhost:8888/auth/realms/opentdf"' opentdf.yaml - yq -i ' - .server.cryptoProvider = { - "type": "standard", - "standard": { - "keys": [ - { - "kid": "r1", - "alg": "rsa:2048", - "private": "kas-private.pem", - "cert": "kas-cert.pem" - }, - { - "kid": "e1", - "alg": "ec:secp256r1", - "private": "kas-ec-private.pem", - "cert": "kas-ec-cert.pem" - } - ] - } - } - ' opentdf.yaml - sudo chmod -R 777 ./keys - working-directory: platform - # - name: Trust the locally issued cert - # run: | - # keytool \ - # -importcert \ - # -storepass changeit \ - # -noprompt \ - # -file localhost.crt \ - # -keystore $JAVA_HOME/lib/security/cacerts \ - # -alias localhost-for-tests - # working-directory: platform/keys - - name: Bring the services up - shell: bash - run: docker compose up -d --wait --wait-timeout 240 - working-directory: platform - - name: Provision keycloak - shell: bash - run: go run ./service provision keycloak - working-directory: platform - - name: Provision fixtures - shell: bash - run: go run ./service provision fixtures - working-directory: platform - - name: Start server in background - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635 - with: - run: | - go run ./service start - wait-on: | - tcp:localhost:8080 - log-output-if: true - wait-for: 90s - working-directory: platform + - name: Get grpcurl shell: bash run: go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.8.9 + - name: Make sure that the platform is up shell: bash run: | + export PATH=$PATH:$(go env GOPATH)/bin grpcurl -plaintext localhost:8080 list && \ grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey @@ -146,7 +73,7 @@ jobs: - name: Encrypt file with otdfctl (no attributes) run: | export PATH=$PATH:$(go env GOPATH)/bin - otdfctl encrypt -o secret.txt.tdf --host http://localhost:8080 --tls-no-verify --with-client-creds-file creds.json secret.txt + otdfctl encrypt -o secret.txt.tdf --host http://localhost:8080 --with-client-creds-file creds.json secret.txt shell: bash - name: Set up uv @@ -161,7 +88,7 @@ jobs: OPENTDF_CLIENT_ID: "opentdf" OPENTDF_CLIENT_SECRET: "secret" OPENTDF_HOSTNAME: "localhost:8080" - OIDC_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token" + OIDC_TOKEN_ENDPOINT: "http://keycloak.opentdf.local:8888/auth/realms/opentdf/protocol/openid-connect/token" OPENTDF_KAS_URL: "http://localhost:8080/kas" INSECURE_SKIP_VERIFY: "TRUE" TEST_OPENTDF_ATTRIBUTE_1: "https://example.net/attr/attr1/value/value1" @@ -178,7 +105,8 @@ jobs: OPENTDF_CLIENT_SECRET: "secret" OPENTDF_PLATFORM_HOST: "localhost:8080" OPENTDF_PLATFORM_URL: "http://localhost:8080" - OIDC_OP_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token" + OPENTDF_KEYCLOAK_HOST: "keycloak.opentdf.local:8888" + OIDC_OP_TOKEN_ENDPOINT: "http://keycloak.opentdf.local:8888/auth/realms/opentdf/protocol/openid-connect/token" OPENTDF_KAS_URL: "http://localhost:8080/kas" INSECURE_SKIP_VERIFY: "TRUE" TEST_OPENTDF_ATTRIBUTE_1: "https://example.net/attr/attr1/value/value1" From 7955fe3df5638287a9eb59356c342067f23d4848 Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 2 Aug 2026 19:50:08 -0400 Subject: [PATCH 2/7] Fix OIDC config discovery bug --- packages/otdf-python/src/otdf_python/sdk_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/otdf-python/src/otdf_python/sdk_builder.py b/packages/otdf-python/src/otdf_python/sdk_builder.py index c320c2d..a02500d 100644 --- a/packages/otdf-python/src/otdf_python/sdk_builder.py +++ b/packages/otdf-python/src/otdf_python/sdk_builder.py @@ -273,7 +273,9 @@ def _discover_token_endpoint_from_platform(self) -> None: ) config_doc = response.json() - configuration = config_doc.get("configuration", {}) + # Newer platform versions return the fields directly at the top + # level instead of nested under a "configuration" key. + configuration = config_doc.get("configuration") or config_doc # Try to get token endpoint from IDP configuration idp_config = configuration.get("idp", {}) From 6e837d7d77b80bf469f779015d1db7ed19a820c1 Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 2 Aug 2026 20:09:52 -0400 Subject: [PATCH 3/7] Fix broken docker compose wait in CI --- .../workflows/platform-integration-test.yaml | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index 1920801..fa538c7 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -36,10 +36,15 @@ jobs: - name: Bring the platform up shell: bash run: | + # Note: intentionally not using `--wait` here. This compose file + # has many one-shot init/provisioning containers (restart: "no"), + # and `docker compose up --wait` unreliably reports failure for + # those even when every container has exited cleanly -- readiness + # is instead confirmed below by polling grpcurl. docker compose \ --file .github/docker-compose.platform.yaml \ --file .github/docker-compose.platform.override.yaml \ - up -d --wait --wait-timeout 360 + up -d - name: Set up go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e @@ -55,8 +60,25 @@ jobs: shell: bash run: | export PATH=$PATH:$(go env GOPATH)/bin - grpcurl -plaintext localhost:8080 list && \ - grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + for i in $(seq 1 60); do + if grpcurl -plaintext localhost:8080 list >/dev/null 2>&1; then + echo "Platform is up after ${i} attempt(s)" + grpcurl -plaintext localhost:8080 list + grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey + exit 0 + fi + sleep 5 + done + echo "::error::Platform did not come up in time" + docker compose \ + --file .github/docker-compose.platform.yaml \ + --file .github/docker-compose.platform.override.yaml \ + ps -a + docker compose \ + --file .github/docker-compose.platform.yaml \ + --file .github/docker-compose.platform.override.yaml \ + logs platform + exit 1 - name: Install otdfctl run: go install github.com/opentdf/otdfctl@v0.28.0 From 240e2fd1b48f982a38795cbbf4370dda6e08cfba Mon Sep 17 00:00:00 2001 From: b-long Date: Sun, 2 Aug 2026 20:37:57 -0400 Subject: [PATCH 4/7] Pin all image references --- .github/docker-compose.platform.env | 8 +++++ .github/docker-compose.platform.override.yaml | 33 +++++++++---------- .github/docker-compose.platform.yaml | 17 ++++++---- .github/keycloak_data.yaml | 14 ++++---- .../workflows/platform-integration-test.yaml | 3 ++ 5 files changed, 45 insertions(+), 30 deletions(-) create mode 100644 .github/docker-compose.platform.env diff --git a/.github/docker-compose.platform.env b/.github/docker-compose.platform.env new file mode 100644 index 0000000..45d2f69 --- /dev/null +++ b/.github/docker-compose.platform.env @@ -0,0 +1,8 @@ +# Single source of truth for the opentdf/platform version pinned in +# docker-compose.platform.override.yaml. Passed to `docker compose` via +# --env-file so PLATFORM_VERSION only has to be declared once. +# +# NanoTDF rewrap (kas ec rewrap) breaks against every platform release +# newer than this that was tried (v0.16.0 through :nightly), so we're +# pinned here for now -- see issue #26. +PLATFORM_VERSION=v0.8.1 diff --git a/.github/docker-compose.platform.override.yaml b/.github/docker-compose.platform.override.yaml index fc9b017..0fadd0d 100644 --- a/.github/docker-compose.platform.override.yaml +++ b/.github/docker-compose.platform.override.yaml @@ -2,15 +2,19 @@ # integration tests in CI (and locally). See issue #26 for the background # on each of these -- briefly: # -# - The platform, its Keycloak provisioning data, and its policy fixtures -# must all come from the SAME git tag. NanoTDF rewrap (kas ec rewrap) -# breaks against every platform release newer than service/v0.8.1 that -# was tried (v0.16.0 through :nightly), independent of otdfctl/SDK -# version, so everything is pinned to that tag for now. -# - Our own keycloak_data.yaml (matching the v0.8.1 realm schema) replaces -# the copy the base compose file downloads from opentdf/platform@main, -# since we need `directAccessGrantsEnabled: true` on the "opentdf" client -# for the OAuth2 password grant used by the PE integration tests. +# - The platform's Keycloak provisioning data and policy fixtures must come +# from the SAME git tag as the platform image (docker-compose.platform.yaml +# already pins the image itself to PLATFORM_VERSION, declared once in +# docker-compose.platform.env -- pass that file via `docker compose +# --env-file`). NanoTDF rewrap (kas ec rewrap) breaks against every +# platform release newer than that tag was tried against (v0.16.0 through +# :nightly), independent of otdfctl/SDK version, so everything is pinned +# for now. +# - Our own keycloak_data.yaml (matching that same version's realm schema) +# replaces the copy the base compose file downloads from +# opentdf/platform@main, since we need `directAccessGrantsEnabled: true` +# on the "opentdf" client for the OAuth2 password grant used by the PE +# integration tests. # - Keycloak's plaintext port is published and opentdf.yaml's issuer is # patched to match, instead of going through Caddy's TLS route. Our # Connect RPC client for KAS (pyqwest-based) cannot disable TLS @@ -18,18 +22,11 @@ # Keycloak endpoints used by the Python SDK/tests need to stay plaintext. # (otdfctl doesn't have this limitation and is fine going through Caddy.) services: - platform-provision-keycloak: - image: registry.opentdf.io/platform:v0.8.1 - platform-provision-fixtures: - image: registry.opentdf.io/platform:v0.8.1 - platform: - image: registry.opentdf.io/platform:v0.8.1 - download-platform-config: - command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/v0.8.1/opentdf-example.yaml'] + command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/opentdf-example.yaml'] download-fixtures: - command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/v0.8.1/service/internal/fixtures/policy_fixtures.yaml'] + command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/service/internal/fixtures/policy_fixtures.yaml'] download-keycloak-data: entrypoint: /bin/sh diff --git a/.github/docker-compose.platform.yaml b/.github/docker-compose.platform.yaml index be3a42a..67d82e2 100644 --- a/.github/docker-compose.platform.yaml +++ b/.github/docker-compose.platform.yaml @@ -1,9 +1,14 @@ # Vendored from https://opentdf.io/quickstart/docker-compose.yaml # (the "Getting the Platform Running" section of -# https://opentdf.io/getting-started#getting-the-platform-running). +# https://opentdf.io/getting-started#getting-the-platform-running), with one +# deliberate deviation from the upstream file: the `platform*` services' +# image tags were changed from the literal `:nightly` to `:${PLATFORM_VERSION}`, +# so the version we're pinned to (see docker-compose.platform.env) is +# declared in exactly one place instead of being overridden after the fact. +# If re-vendoring this file from upstream, reapply that change. # -# See docker-compose.platform.override.yaml for the local overrides needed -# to make this usable in CI (see issue #26). +# See docker-compose.platform.override.yaml for the other local overrides +# needed to make this usable in CI (see issue #26). networks: default: name: opentdf_platform @@ -193,7 +198,7 @@ services: # Provision Keycloak with initial configuration platform-provision-keycloak: - image: registry.opentdf.io/platform:nightly + image: registry.opentdf.io/platform:${PLATFORM_VERSION} command: ["provision", "keycloak", "-e", "https://keycloak.opentdf.local:9443/auth", "-f", "/configs/keycloak_data.yaml"] depends_on: keycloak: @@ -235,7 +240,7 @@ services: # Add sample attributes and metadata platform-provision-fixtures: - image: registry.opentdf.io/platform:nightly + image: registry.opentdf.io/platform:${PLATFORM_VERSION} command: ["provision", "fixtures", "--config-file", "/configs/opentdf.yaml"] working_dir: /configs depends_on: @@ -284,7 +289,7 @@ services: # Main OpenTDF Platform server platform: - image: registry.opentdf.io/platform:nightly + image: registry.opentdf.io/platform:${PLATFORM_VERSION} command: ["start", "--config-file", "/configs/opentdf.yaml"] depends_on: platform-provision-fixtures: diff --git a/.github/keycloak_data.yaml b/.github/keycloak_data.yaml index 01c187a..55a18ed 100644 --- a/.github/keycloak_data.yaml +++ b/.github/keycloak_data.yaml @@ -1,12 +1,14 @@ # Vendored from -# https://raw.githubusercontent.com/opentdf/platform/service/v0.8.1/service/cmd/keycloak_data.yaml -# with one change: `directAccessGrantsEnabled: true` added to the "opentdf" -# client, which our integration tests need for the OAuth2 password grant +# https://raw.githubusercontent.com/opentdf/platform/service//service/cmd/keycloak_data.yaml +# (see PLATFORM_VERSION in docker-compose.platform.env) with one change: +# `directAccessGrantsEnabled: true` added to the "opentdf" client, which our +# integration tests need for the OAuth2 password grant # (see tests/integration/support_sdk.py::get_user_access_token). # -# This must stay pinned to the same platform version/tag as the images in -# docker-compose.platform.override.yaml -- the realm schema this file uses -# is not compatible with newer platform releases (see issue #26). +# This is a static file, so if PLATFORM_VERSION is ever bumped, re-fetch it +# from the URL above (with that version substituted in) and re-apply the +# change above -- the realm schema here is not compatible with newer +# platform releases (see issue #26). baseUrl: &baseUrl http://localhost:8888 serverBaseUrl: &serverBaseUrl http://localhost:8080 customAudMapper: &customAudMapper diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index fa538c7..af39dd8 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -44,6 +44,7 @@ jobs: docker compose \ --file .github/docker-compose.platform.yaml \ --file .github/docker-compose.platform.override.yaml \ + --env-file .github/docker-compose.platform.env \ up -d - name: Set up go @@ -73,10 +74,12 @@ jobs: docker compose \ --file .github/docker-compose.platform.yaml \ --file .github/docker-compose.platform.override.yaml \ + --env-file .github/docker-compose.platform.env \ ps -a docker compose \ --file .github/docker-compose.platform.yaml \ --file .github/docker-compose.platform.override.yaml \ + --env-file .github/docker-compose.platform.env \ logs platform exit 1 From f77cdfd05d0ad06e556eeba5b92b267f192dac0d Mon Sep 17 00:00:00 2001 From: b-long Date: Mon, 3 Aug 2026 21:41:21 -0400 Subject: [PATCH 5/7] pin remaining platform fetches to PLATFORM_VERSION --- .github/docker-compose.platform.override.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/docker-compose.platform.override.yaml b/.github/docker-compose.platform.override.yaml index 0fadd0d..fa8866a 100644 --- a/.github/docker-compose.platform.override.yaml +++ b/.github/docker-compose.platform.override.yaml @@ -21,6 +21,16 @@ # certificate verification for https:// URLs, so the platform and # Keycloak endpoints used by the Python SDK/tests need to stay plaintext. # (otdfctl doesn't have this limitation and is fine going through Caddy.) +# - Every file the base compose file fetches from opentdf/platform@main is +# repinned here to the same git tag as PLATFORM_VERSION (`service/`), +# so nothing in this pipeline tracks main unpinned -- see download-init-script +# below. The one exception is generate-pqc-keys: the PQC keygen tool +# (service/cmd/keygen) doesn't exist yet at service/${PLATFORM_VERSION}, and +# the opentdf.yaml pinned to that tag only references the RSA/EC keys that +# generate-keys already creates, not PQC/hybrid ones -- so its output isn't +# consumed at this pin. It's turned into a no-op below instead of being +# pointed at a ref that doesn't have the tool. Revisit both when +# PLATFORM_VERSION is next bumped. services: download-platform-config: command: ['wget', '-O', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/opentdf-example.yaml'] @@ -28,6 +38,18 @@ services: download-fixtures: command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/service/internal/fixtures/policy_fixtures.yaml'] + download-init-script: + command: ['wget', '-O', '/configs/init-temp-keys.sh', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/.github/scripts/init-temp-keys.sh'] + + generate-pqc-keys: + image: alpine:3.23 + entrypoint: /bin/sh + command: + - -c + - | + echo "Skipping PQC key generation: service/cmd/keygen doesn't exist at service/${PLATFORM_VERSION}, and its output isn't referenced by opentdf.yaml at that pin." + restart: "no" + download-keycloak-data: entrypoint: /bin/sh command: From 2b0b63d0885cc1ef66f3750b519b66c21cecc2c6 Mon Sep 17 00:00:00 2001 From: b-long Date: Mon, 3 Aug 2026 21:55:50 -0400 Subject: [PATCH 6/7] stop vendoring keycloak_data.yaml, patch with yq --- .github/docker-compose.platform.override.yaml | 23 ++-- .github/keycloak_data.yaml | 125 ------------------ .../workflows/platform-integration-test.yaml | 6 +- 3 files changed, 15 insertions(+), 139 deletions(-) delete mode 100644 .github/keycloak_data.yaml diff --git a/.github/docker-compose.platform.override.yaml b/.github/docker-compose.platform.override.yaml index fa8866a..bbc8733 100644 --- a/.github/docker-compose.platform.override.yaml +++ b/.github/docker-compose.platform.override.yaml @@ -10,11 +10,14 @@ # platform release newer than that tag was tried against (v0.16.0 through # :nightly), independent of otdfctl/SDK version, so everything is pinned # for now. -# - Our own keycloak_data.yaml (matching that same version's realm schema) -# replaces the copy the base compose file downloads from -# opentdf/platform@main, since we need `directAccessGrantsEnabled: true` -# on the "opentdf" client for the OAuth2 password grant used by the PE -# integration tests. +# - The Keycloak provisioning data must come from the SAME git tag as the +# platform image (see download-keycloak-data below), patched with +# `directAccessGrantsEnabled: true` on the "opentdf" client via `yq`, which +# our integration tests need for the OAuth2 password grant (see +# tests/integration/support_sdk.py::get_user_access_token). This mirrors +# how the pre-docker-compose CI patched the same file with `yq` after +# checking out the platform repo at this tag -- see git history of +# platform-integration-test.yaml before this pipeline used docker compose. # - Keycloak's plaintext port is published and opentdf.yaml's issuer is # patched to match, instead of going through Caddy's TLS route. Our # Connect RPC client for KAS (pyqwest-based) cannot disable TLS @@ -51,15 +54,13 @@ services: restart: "no" download-keycloak-data: - entrypoint: /bin/sh command: - -c - | - cp /custom/keycloak_data.yaml /configs/keycloak_data.yaml - echo "Copied local keycloak_data.yaml into /configs" - volumes: - - configs:/configs - - ./keycloak_data.yaml:/custom/keycloak_data.yaml:ro + apk add --no-cache yq + wget -O /configs/keycloak_data.yaml 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/service/cmd/keycloak_data.yaml' + yq -i '.realms[0].clients[0].client.directAccessGrantsEnabled = true' /configs/keycloak_data.yaml + echo "Downloaded keycloak_data.yaml (service/${PLATFORM_VERSION}) and patched directAccessGrantsEnabled for the opentdf client" keycloak: ports: diff --git a/.github/keycloak_data.yaml b/.github/keycloak_data.yaml deleted file mode 100644 index 55a18ed..0000000 --- a/.github/keycloak_data.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# Vendored from -# https://raw.githubusercontent.com/opentdf/platform/service//service/cmd/keycloak_data.yaml -# (see PLATFORM_VERSION in docker-compose.platform.env) with one change: -# `directAccessGrantsEnabled: true` added to the "opentdf" client, which our -# integration tests need for the OAuth2 password grant -# (see tests/integration/support_sdk.py::get_user_access_token). -# -# This is a static file, so if PLATFORM_VERSION is ever bumped, re-fetch it -# from the URL above (with that version substituted in) and re-apply the -# change above -- the realm schema here is not compatible with newer -# platform releases (see issue #26). -baseUrl: &baseUrl http://localhost:8888 -serverBaseUrl: &serverBaseUrl http://localhost:8080 -customAudMapper: &customAudMapper - name: audience-mapper - protocol: openid-connect - protocolMapper: oidc-audience-mapper - config: - included.custom.audience: *serverBaseUrl - access.token.claim: "true" - id.token.claim: "true" -realms: - - realm_repepresentation: - realm: opentdf - enabled: true - custom_realm_roles: - - name: opentdf-admin - - name: opentdf-standard - custom_client_roles: - tdf-entity-resolution: - - name: entity-resolution-test-role - custom_groups: - - name: mygroup - attributes: - mygroupattribute: - - mygroupvalue - clients: - - client: - clientID: opentdf - enabled: true - name: opentdf - serviceAccountsEnabled: true - clientAuthenticatorType: client-secret - secret: secret - protocolMappers: - - *customAudMapper - directAccessGrantsEnabled: true - sa_realm_roles: - - opentdf-admin - copies: 10 - - client: - clientID: opentdf-sdk - enabled: true - name: opentdf-sdk - serviceAccountsEnabled: true - clientAuthenticatorType: client-secret - secret: secret - protocolMappers: - - *customAudMapper - sa_realm_roles: - - opentdf-standard - - client: - clientID: tdf-entity-resolution - enabled: true - name: tdf-entity-resolution - serviceAccountsEnabled: true - clientAuthenticatorType: client-secret - secret: secret - protocolMappers: - - *customAudMapper - sa_client_roles: - realm-management: - - view-clients - - query-clients - - view-users - - query-users - - client: - clientID: tdf-authorization-svc - enabled: true - name: tdf-authorization-svc - serviceAccountsEnabled: true - clientAuthenticatorType: client-secret - secret: secret - protocolMappers: - - *customAudMapper - - client: - clientID: cli-client - enabled: true - name: cli-client - serviceAccountsEnabled: false - publicClient: true - redirectUris: - - 'http://localhost:*' - protocolMappers: - - *customAudMapper - users: - - username: sample-user - enabled: true - firstName: sample - lastName: user - email: sampleuser@sample.com - credentials: - - value: testuser123 - type: password - attributes: - superhero_name: - - thor - superhero_group: - - avengers - groups: - - mygroup - realmRoles: - - opentdf-admin - clientRoles: - realm-management: - - view-clients - - query-clients - - view-users - - query-users - tdf-entity-resolution: - - entity-resolution-test-role - copies: 10 - token_exchanges: - - start_client: opentdf - target_client: opentdf-sdk diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml index af39dd8..868a129 100644 --- a/.github/workflows/platform-integration-test.yaml +++ b/.github/workflows/platform-integration-test.yaml @@ -3,9 +3,9 @@ # instead of building the platform from source (see issue #26). # # docker-compose.platform.yaml is a vendored copy of the quickstart's -# docker-compose.yaml. docker-compose.platform.override.yaml and -# keycloak_data.yaml carry the local changes needed to make it usable here -# -- see the comments in those files for why each one is needed. +# docker-compose.yaml. docker-compose.platform.override.yaml carries the +# local changes needed to make it usable here -- see the comments in that +# file for why each one is needed. name: "NEW: Platform Integration testing" on: From eb2cb8a48c2a96784fea06f6c8baa188055b13f7 Mon Sep 17 00:00:00 2001 From: b-long Date: Mon, 3 Aug 2026 22:01:20 -0400 Subject: [PATCH 7/7] Fix entrypoint ( /bin/sh ) shell form --- .github/docker-compose.platform.override.yaml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/docker-compose.platform.override.yaml b/.github/docker-compose.platform.override.yaml index bbc8733..95822ac 100644 --- a/.github/docker-compose.platform.override.yaml +++ b/.github/docker-compose.platform.override.yaml @@ -10,13 +10,12 @@ # platform release newer than that tag was tried against (v0.16.0 through # :nightly), independent of otdfctl/SDK version, so everything is pinned # for now. -# - The Keycloak provisioning data must come from the SAME git tag as the -# platform image (see download-keycloak-data below), patched with -# `directAccessGrantsEnabled: true` on the "opentdf" client via `yq`, which -# our integration tests need for the OAuth2 password grant (see -# tests/integration/support_sdk.py::get_user_access_token). This mirrors -# how the pre-docker-compose CI patched the same file with `yq` after -# checking out the platform repo at this tag -- see git history of +# - The Keycloak provisioning data (download-keycloak-data below) is also +# patched with `directAccessGrantsEnabled: true` on the "opentdf" client +# via `yq`, which our integration tests need for the OAuth2 password grant +# (see tests/integration/support_sdk.py::get_user_access_token). This +# mirrors how the pre-docker-compose CI patched the same file with `yq` +# after checking out the platform repo at this tag -- see git history of # platform-integration-test.yaml before this pipeline used docker compose. # - Keycloak's plaintext port is published and opentdf.yaml's issuer is # patched to match, instead of going through Caddy's TLS route. Our @@ -42,7 +41,14 @@ services: command: ['wget', '-O', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/service/internal/fixtures/policy_fixtures.yaml'] download-init-script: - command: ['wget', '-O', '/configs/init-temp-keys.sh', 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/.github/scripts/init-temp-keys.sh'] + # Unlike download-platform-config/download-fixtures below, the base + # service sets `entrypoint: /bin/sh`, so the command must be shell + # script text (`-c ...`), not an exec-form argv list -- otherwise sh + # tries to run "wget" as a script file and exits 2. + command: + - -c + - | + wget -O /configs/init-temp-keys.sh 'https://raw.githubusercontent.com/opentdf/platform/service/${PLATFORM_VERSION}/.github/scripts/init-temp-keys.sh' generate-pqc-keys: image: alpine:3.23