Beta to main

.github/workflows/pr-checks.yml

@@ -9,7 +9,7 @@ jobs:
     uses: nullplatform/actions-nullplatform/.github/workflows/pr-checks-go.yml@main
     with:
       working-directory: k8s/log/kube-logger-go
-      go-version: '1.25'
+      go-version: '1.25.13'
 
   shellcheck:
     uses: nullplatform/actions-nullplatform/.github/workflows/shellcheck.yml@main
@@ -19,6 +19,8 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true # catalog-tracing-sh — the tracing SDK the tests exercise
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y bats jq

.github/workflows/publish-images.yml

@@ -0,0 +1,169 @@
+name: publish-images
+
+# Publishes every scope's worker image to ECR Public on each semver tag, then
+# registers each as an oci_image platform artifact (visible-to organization=*).
+# Same mold as scopes-lambda (publish-image.yml), fanned out to the 3 images:
+#
+#   scopes/containers            <- k8s/            (base; FROM worker-bridge + tooling)
+#   scopes/scheduled-task        <- scheduled_task/ (leaner)
+#   scopes/containers-datadog    <- containers + datadog/  overlay (metric)
+#
+# The datadog overlay is FROM the containers base (which bakes the whole repo
+# into /app/pkg), so it must be pushed first — its build `needs: containers` and
+# passes BASE_VERSION so its Dockerfile can FROM containers:<tag>.
+# (azure / azure-aro are pure config overlays the tofu modules configure at
+# install time via NP_OVERRIDES_PATH, so they need no image of their own.)
+#
+# Explicit job per image on purpose: a matrixed job that calls the reusable
+# workflow collapses `image_digest` to a single value (matrix outputs overwrite
+# each other), which would register every artifact against the same image.
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  id-token: write    # OIDC against AWS
+  contents: write    # create/update the GitHub release with artifact metadata
+
+jobs:
+  # ── containers (base) ──────────────────────────────────────────────────────
+  containers:
+    uses: nullplatform/actions-nullplatform/.github/workflows/docker-build-push-ecr.yml@main
+    with:
+      image_name: scopes/containers
+      context: .
+      submodules: true
+      dockerfile: docker/containers.Dockerfile
+      tag: ${{ github.ref_name }}
+    secrets:
+      aws_role_arn: ${{ secrets.AWS_ROLE_ARN_ECR_PUSH }}
+
+  register-containers:
+    name: Register containers artifact
+    needs: containers
+    runs-on: ubuntu-24.04
+    env:
+      NULLPLATFORM_API_KEY: ${{ secrets.NP_API_KEY }}
+      NP_ARTIFACT_NRN: ${{ vars.NP_ARTIFACT_NRN }}
+    steps:
+      - name: Install np CLI (alpha-packages build)
+        run: curl -s https://cli.nullplatform.com/install.sh | VERSION=alpha-packages sh
+      - name: Register scopes/containers image artifact (visible to everyone)
+        run: |
+          np artifact create \
+            --nrn "$NP_ARTIFACT_NRN" \
+            --type oci_image \
+            --registry public.ecr.aws \
+            --repository nullplatform/scopes/containers \
+            --digest "${{ needs.containers.outputs.image_digest }}" \
+            --visible-to "organization=*"
+
+  # ── scheduled-task (standalone) ────────────────────────────────────────────
+  scheduled-task:
+    uses: nullplatform/actions-nullplatform/.github/workflows/docker-build-push-ecr.yml@main
+    with:
+      image_name: scopes/scheduled-task
+      context: .
+      submodules: true
+      dockerfile: docker/scheduled-task.Dockerfile
+      tag: ${{ github.ref_name }}
+    secrets:
+      aws_role_arn: ${{ secrets.AWS_ROLE_ARN_ECR_PUSH }}
+
+  register-scheduled-task:
+    name: Register scheduled-task artifact
+    needs: scheduled-task
+    runs-on: ubuntu-24.04
+    env:
+      NULLPLATFORM_API_KEY: ${{ secrets.NP_API_KEY }}
+      NP_ARTIFACT_NRN: ${{ vars.NP_ARTIFACT_NRN }}
+    steps:
+      - name: Install np CLI (alpha-packages build)
+        run: curl -s https://cli.nullplatform.com/install.sh | VERSION=alpha-packages sh
+      - name: Register scopes/scheduled-task image artifact (visible to everyone)
+        run: |
+          np artifact create \
+            --nrn "$NP_ARTIFACT_NRN" \
+            --type oci_image \
+            --registry public.ecr.aws \
+            --repository nullplatform/scopes/scheduled-task \
+            --digest "${{ needs.scheduled-task.outputs.image_digest }}" \
+            --visible-to "organization=*"
+
+  # ── containers-datadog (overlay) ───────────────────────────────────────────
+  containers-datadog:
+    needs: containers
+    uses: nullplatform/actions-nullplatform/.github/workflows/docker-build-push-ecr.yml@main
+    with:
+      image_name: scopes/containers-datadog
+      context: .
+      submodules: true
+      dockerfile: docker/containers-datadog.Dockerfile
+      tag: ${{ github.ref_name }}
+      build_args: BASE_VERSION=${{ github.ref_name }}
+    secrets:
+      aws_role_arn: ${{ secrets.AWS_ROLE_ARN_ECR_PUSH }}
+
+  register-containers-datadog:
+    name: Register containers-datadog artifact
+    needs: containers-datadog
+    runs-on: ubuntu-24.04
+    env:
+      NULLPLATFORM_API_KEY: ${{ secrets.NP_API_KEY }}
+      NP_ARTIFACT_NRN: ${{ vars.NP_ARTIFACT_NRN }}
+    steps:
+      - name: Install np CLI (alpha-packages build)
+        run: curl -s https://cli.nullplatform.com/install.sh | VERSION=alpha-packages sh
+      - name: Register scopes/containers-datadog image artifact (visible to everyone)
+        run: |
+          np artifact create \
+            --nrn "$NP_ARTIFACT_NRN" \
+            --type oci_image \
+            --registry public.ecr.aws \
+            --repository nullplatform/scopes/containers-datadog \
+            --digest "${{ needs.containers-datadog.outputs.image_digest }}" \
+            --visible-to "organization=*"
+
+  # ── GitHub release with artifact metadata ──────────────────────────────────
+  # Tags here are human-pushed and previously produced no GitHub release at
+  # all, so the published digests were invisible to consumers. Upsert a release
+  # for the tag whose body carries every image's digest and copyable pinned
+  # reference. Runs after the registrations so the table reflects what was
+  # actually published; the digest check keeps re-runs idempotent.
+  finalize-release:
+    name: Publish release with artifact metadata
+    needs: [containers, scheduled-task, containers-datadog, register-containers, register-scheduled-task, register-containers-datadog]
+    runs-on: ubuntu-24.04
+    env:
+      GH_TOKEN: ${{ github.token }}
+      TAG: ${{ github.ref_name }}
+      REGISTRY: public.ecr.aws/nullplatform
+      DIGEST_CONTAINERS: ${{ needs.containers.outputs.image_digest }}
+      DIGEST_SCHEDULED: ${{ needs.scheduled-task.outputs.image_digest }}
+      DIGEST_DATADOG: ${{ needs.containers-datadog.outputs.image_digest }}
+    steps:
+      - name: Upsert release with artifact table
+        run: |
+          SECTION=$(printf '## Artifacts\n\n| Image | Digest | Pinned reference |\n|---|---|---|\n| `%s:%s` | `%s` | `%s@%s` |\n| `%s:%s` | `%s` | `%s@%s` |\n| `%s:%s` | `%s` | `%s@%s` |' \
+            "$REGISTRY/scopes/containers" "$TAG" "$DIGEST_CONTAINERS" "$REGISTRY/scopes/containers" "$DIGEST_CONTAINERS" \
+            "$REGISTRY/scopes/scheduled-task" "$TAG" "$DIGEST_SCHEDULED" "$REGISTRY/scopes/scheduled-task" "$DIGEST_SCHEDULED" \
+            "$REGISTRY/scopes/containers-datadog" "$TAG" "$DIGEST_DATADOG" "$REGISTRY/scopes/containers-datadog" "$DIGEST_DATADOG")
+
+          # Drafts are not resolvable via releases/tags/:tag — list and filter.
+          RELEASE_ID=$(gh api "repos/$GITHUB_REPOSITORY/releases" --paginate \
+            --jq "[.[] | select(.tag_name==\"$TAG\")][0].id // empty")
+
+          if [ -z "$RELEASE_ID" ]; then
+            gh release create "$TAG" --title "$TAG" --notes "$SECTION" --verify-tag
+            echo "created release $TAG"
+          else
+            BODY=$(gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" --jq '.body // ""')
+            if printf '%s' "$BODY" | grep -qF "$DIGEST_CONTAINERS"; then
+              echo "release $TAG already carries these digests"
+            else
+              printf '%s\n\n%s' "$BODY" "$SECTION" > body.md
+              gh release edit "$TAG" --draft=false --notes-file body.md
+              echo "updated release $TAG"
+            fi
+          fi

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "vendor/catalog-tracing-sh"]
+	path = vendor/catalog-tracing-sh
+	url = https://github.com/nullplatform/catalog-tracing-sh.git
+	branch = main

CHANGELOG.md

@@ -5,7 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [1.16.0] - 2026-09-02
+- k8s scopes now show live, step-by-step progress in the dashboard when a scope is created, updated or deleted, and for every deployment action (deploy, switch traffic, finalize, rollback, delete, diagnose, kill instance, restart pods, pause/resume autoscaling, set instance count): each step with its duration, and long waits (load balancer, DNS, instance health) saying what they are waiting for. Requires `NP_API_KEY` on the agent; without it everything works as before
+- Failed k8s deployments now explain why on the step that failed, and what to do about it: image pull errors with the registry's message, out-of-memory kills, crash exit codes with the application's last log lines, and failing health checks with the path and response detected
+- k8s **diagnose** now shows what each check found (summary, severity, affected pods, recommended action) instead of a list of green steps
+- k8s scopes can now pin the traffic-manager sidecar version cluster-wide via the container-orchestration provider's `traffic_manager.version`, instead of only per-scope
+- Publish containers and scheduled task scopes as docker images
+- Remove unused cloudwatch annotations from deployment objects
+- Fix: log queries on k8s scopes now return the time range that was selected, instead of the most recent lines whatever range was chosen
+- Fix: paging through logs on k8s scopes no longer repeats lines already shown, and now reaches the end of the selected range
+- Fix: diagnose on k8s scopes no longer fails to publish its results when a check collects application logs
 
 ## [1.15.1] - 2026-08-12
 - Fix: gRPC additional ports on k8s scopes now leave the declared port free for the application, so a gRPC server can bind the port configured in the scope instead of failing to start with "address already in use". gRPC ports now work the same way HTTP ones already did

azure-aro/values.yaml

@@ -1,5 +1,6 @@
 configuration:
   DNS_TYPE: azure
+  K8S_FLAVOR: aro
   USE_ACCOUNT_SLUG: false
   IMAGE_PULL_SECRETS:
     ENABLED: false

azure/values.yaml

@@ -1,5 +1,6 @@
 configuration:
   DNS_TYPE: azure
+  K8S_FLAVOR: aks
   USE_ACCOUNT_SLUG: false
   IMAGE_PULL_SECRETS:
     ENABLED: false

docker/containers-datadog.Dockerfile

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1
+#
+# containers-datadog — the containers scope with the datadog overlay baked in
+# (the datadog/ folder overrides the metric step). The containers base already
+# COPYied the whole repo into /app/pkg (incl. datadog/), so this only layers the
+# overlay onto the service-path. datadog uses jq + curl, already in the base.
+ARG BASE_VERSION
+FROM public.ecr.aws/nullplatform/scopes/containers:${BASE_VERSION}
+
+ENV NP_PACKAGE_NAME=containers-datadog \
+    NP_OVERRIDES_PATH=/app/pkg/datadog

docker/containers.Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+#
+# containers (k8s) scope image — the full Kubernetes scope on the lean gRPC
+# worker bridge. The bridge dials over gRPC and runs the repo's bash entrypoint
+# on each action; this image adds the cloud tooling the k8s steps call and bakes
+# the whole repo in, so the package-exec channel needs no cmdline.
+#
+# NP_SERVICE_PATH=k8s + NP_SCOPE_ENTRYPOINT=<repo>/entrypoint mirrors the classic
+# `.../scopes/entrypoint --service-path=k8s` the git-clone model used.
+FROM public.ecr.aws/nullplatform/scopes/worker-bridge:1.0.0
+
+# apk tooling the k8s steps call. bash, jq, np, base64, curl, ca-certs ship in
+# the base. aws-cli 2.x, gomplate and yq are packaged on alpine.
+RUN apk add --no-cache aws-cli gomplate yq
+
+# Pinned binaries not reliably packaged on alpine: OpenTofu, kubectl, helm.
+# NOTE: review/pin these versions to what the scopes actually target.
+ARG TARGETARCH
+ARG TOFU_VERSION=1.10.6
+ARG KUBECTL_VERSION=1.30.4
+ARG HELM_VERSION=3.15.4
+RUN set -eux; \
+    curl -fsSL "https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}/tofu_${TOFU_VERSION}_linux_${TARGETARCH}.tar.gz" \
+      | tar -xz -C /usr/local/bin tofu; \
+    curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl"; \
+    chmod +x /usr/local/bin/kubectl; \
+    curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" | tar -xz -C /tmp; \
+    mv "/tmp/linux-${TARGETARCH}/helm" /usr/local/bin/helm; \
+    rm -rf "/tmp/linux-${TARGETARCH}"; \
+    tofu version && kubectl version --client && helm version --short
+
+# Bake the whole repo in; the overlays (containers-azure, -datadog, -aro) are
+# FROM this image and only flip NP_OVERRIDES_PATH — no re-copy needed.
+COPY . /app/pkg
+ENV NP_PACKAGE_NAME=containers \
+    NP_SERVICE_PATH=/app/pkg/k8s \
+    NP_SCOPE_ENTRYPOINT=/app/pkg/entrypoint

docker/scheduled-task.Dockerfile

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1
+#
+# scheduled-task scope image — the scheduled_task scope. Leaner than containers:
+# its steps only reach for kubectl + gomplate (bash/jq/np ship in the base).
+FROM public.ecr.aws/nullplatform/scopes/worker-bridge:1.0.0
+
+RUN apk add --no-cache gomplate
+
+ARG TARGETARCH
+ARG KUBECTL_VERSION=1.30.4
+RUN curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" \
+ && chmod +x /usr/local/bin/kubectl \
+ && kubectl version --client
+
+COPY . /app/pkg
+ENV NP_PACKAGE_NAME=scheduled-task \
+    NP_SERVICE_PATH=/app/pkg/scheduled_task \
+    NP_SCOPE_ENTRYPOINT=/app/pkg/entrypoint

k8s/apply_templates

@@ -31,9 +31,39 @@ while IFS= read -r TEMPLATE_FILE; do
        IGNORE_NOT_FOUND="--ignore-not-found=true"
      fi
 
-     if ! kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND; then
-       log error "   ❌ Failed to apply"
+     TRACE_STEP_KEY=""
+     if [[ "$ACTION" == "apply" ]] && command -v np_scope_step_begin >/dev/null 2>&1; then
+       case "$FILENAME" in
+         deployment-*) TRACE_STEP_KEY="create-deployment" ;;
+         secret-*)     TRACE_STEP_KEY="create-secret" ;;
+         scaling-*)    TRACE_STEP_KEY="create-hpa" ;;
+         service-*)    TRACE_STEP_KEY="create-service" ;;
+         pdb-*)        TRACE_STEP_KEY="create-pod-disruption-budget" ;;
+         ingress-*)
+           if [[ -n "${DEPLOYMENT_ID:-}" ]]; then
+             TRACE_STEP_KEY="configure-ingress"
+           else
+             TRACE_STEP_KEY="create-ingress"
+           fi ;;
+       esac
+       [[ -n "$TRACE_STEP_KEY" ]] && np_scope_step_begin "$TRACE_STEP_KEY"
      fi
+
+     if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND 2>&1); then
+       [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT"
+       [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0
+       if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then
+         np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT"
+       fi
+       if [[ "$ACTION" == "delete" ]] && command -v np_scope_k8s_deleted >/dev/null 2>&1; then
+         np_scope_k8s_deleted "${K8S_NAMESPACE:-}" "$KUBECTL_OUT"
+       fi
+     else
+       [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" >&2
+       log error "   ❌ Failed to apply $FILENAME${KUBECTL_OUT:+: $KUBECTL_OUT}"
+       [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 1
+     fi
+     TRACE_STEP_KEY=""
    fi
 
    DEST_DIR="${BASE_DIR}/$ACTION"
@@ -52,4 +82,23 @@ if [[ "$DRY_RUN" == "true" ]]; then
   exit 1
 fi
 
+if [[ "${TRACE_TRAFFIC_SWITCH:-false}" == "true" ]] && [[ -n "${CONTEXT:-}" ]] \
+  && command -v np_scope_explain >/dev/null 2>&1; then
+  TRAFFIC_TO=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.desired_switched_traffic // 100')
+  TRAFFIC_FROM=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.switched_traffic // 0')
+  if [[ "$TRAFFIC_TO" =~ ^[0-9]+$ ]] && [[ "$TRAFFIC_FROM" =~ ^[0-9]+$ ]]; then
+    np_scope_labels "deployment.id=${DEPLOYMENT_ID:-}" "scope.id=${SCOPE_ID:-}" "action=traffic-switch"
+    np_scope_explain --title "Switch traffic for deployment ${DEPLOYMENT_ID:-}" \
+      --what "Switching blue/green traffic for deployment ${DEPLOYMENT_ID:-} from ${TRAFFIC_FROM}% to ${TRAFFIC_TO}%"
+    np_scope_affordance "{\"kind\":\"traffic-switch\",\"deployment_id\":\"${DEPLOYMENT_ID:-}\",\"current_traffic\":$TRAFFIC_TO,\"new_traffic\":$TRAFFIC_TO,\"old_traffic\":$((100 - TRAFFIC_TO)),\"target_traffic\":100}"
+    np_scope_input traffic "{\"from\":$TRAFFIC_FROM,\"desired\":$TRAFFIC_TO}"
+    np_scope_output traffic "{\"switched\":$TRAFFIC_TO}"
+    np_scope_progress "$TRAFFIC_TO" 100 percent
+  fi
+fi
+
+if command -v np_trace_flush >/dev/null 2>&1; then
+  NP_TRACE_FLUSH_TIMEOUT=5 np_trace_flush
+fi
+
 source "$SERVICE_PATH/backup/backup_templates" --action="$ACTION" --files "${APPLIED_FILES[@]}"

k8s/deployment/build_context

@@ -139,10 +139,13 @@ fi
 
 SCOPE_TRAFFIC_PROTOCOL=$(echo "$CONTEXT" | jq -r .scope.capabilities.protocol)
 
-TRAFFIC_CONTAINER_VERSION="latest"
-
 if [[ "$SCOPE_TRAFFIC_PROTOCOL" == "web_sockets" ]]; then
   TRAFFIC_CONTAINER_VERSION="websocket2"
+else
+  TRAFFIC_CONTAINER_VERSION=$(get_config_value \
+    --provider '.providers["container-orchestration"].traffic_manager.version' \
+    --default "latest"
+  )
 fi
 
 TRAFFIC_CONTAINER_IMAGE=$(get_config_value \