diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eae3cca..69101c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Add a "Run once" option to scheduled task scopes: the task runs a single time when the scope is deployed instead of on a recurring schedule, and the deployment waits for it to finish ## [1.15.0] - 2026-08-10 - Fix: **finalize** and **rollback** on blue/green k8s scopes now wait until the load balancer sends all traffic to the surviving deployment before deleting the other one, preventing the 5xx window that happened when it was deleted mid-switch (these actions may take slightly longer as a result) diff --git a/scheduled_task/deployment/build_deployment b/scheduled_task/deployment/build_deployment index c64f5780..31b9b557 100644 --- a/scheduled_task/deployment/build_deployment +++ b/scheduled_task/deployment/build_deployment @@ -5,7 +5,15 @@ SECRET_PATH="$OUTPUT_DIR/secret-$SCOPE_ID-$DEPLOYMENT_ID.yaml" SECRET_FILES_PATH="$OUTPUT_DIR/secret-files-$SCOPE_ID-$DEPLOYMENT_ID.yaml" CONTEXT_PATH="$OUTPUT_DIR/context-$SCOPE_ID.json" -echo "$CONTEXT" | jq --arg replicas "$REPLICAS" '. + {replicas: $replicas}' > "$CONTEXT_PATH" +# In "run-once" mode the template renders a Job with activeDeadlineSeconds set to +# this value, so the Job self-terminates at the same deadline the deployment wait +# gives up at. Ignored by the CronJob branch. +JOB_WAIT_TIMEOUT="${JOB_WAIT_TIMEOUT:-600}" + +echo "$CONTEXT" | jq \ + --arg replicas "$REPLICAS" \ + --argjson job_wait_timeout "$JOB_WAIT_TIMEOUT" \ + '. + {replicas: $replicas, job_wait_timeout: $job_wait_timeout}' > "$CONTEXT_PATH" echo "Building Template: $DEPLOYMENT_TEMPLATE to $DEPLOYMENT_PATH" diff --git a/scheduled_task/deployment/templates/deployment.yaml.tpl b/scheduled_task/deployment/templates/deployment.yaml.tpl index c556acfd..b086ffee 100644 --- a/scheduled_task/deployment/templates/deployment.yaml.tpl +++ b/scheduled_task/deployment/templates/deployment.yaml.tpl @@ -1,5 +1,5 @@ apiVersion: batch/v1 -kind: CronJob +kind: {{ if eq .scope.capabilities.cron "run-once" }}Job{{ else }}CronJob{{ end }} metadata: name: job-{{ .scope.id }}-{{ .deployment.id }} namespace: {{ .k8s_namespace }} @@ -31,6 +31,141 @@ metadata: {{- end }} {{- end }} spec: +{{- if eq .scope.capabilities.cron "run-once" }} + backoffLimit: {{ .scope.capabilities.retries }} + activeDeadlineSeconds: {{ .job_wait_timeout }} + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + name: d-{{ .scope.id }}-{{ .deployment.id }} + app.kubernetes.io/part-of: {{ .component }} + nullplatform: "true" + account: "{{ .account.slug }}" + account_id: "{{ .account.id }}" + namespace: "{{ .namespace.slug }}" + namespace_id: "{{ .namespace.id }}" + application: "{{ .application.slug }}" + application_id: "{{ .application.id }}" + scope: "{{ .scope.slug }}" + scope_id: "{{ .scope.id }}" + deployment_id: "{{ .deployment.id }}" + {{- $global := index .k8s_modifiers "global" }} + {{- if $global }} + {{- $labels := index $global "labels" }} + {{- if $labels }} +{{ data.ToYAML $labels | indent 8 }} + {{- end }} + {{- end }} + {{- $deployment := index .k8s_modifiers "deployment" }} + {{- if $deployment }} + {{- $labels := index $deployment "labels" }} + {{- if $labels }} +{{ data.ToYAML $labels | indent 8 }} + {{- end }} + {{- end }} + annotations: + nullplatform.logs.cloudwatch: 'true' + nullplatform.logs.cloudwatch.log_group_name: {{ .namespace.slug }}.{{ .application.slug }} + nullplatform.logs.cloudwatch.log_stream_log_retention_days: '7' + nullplatform.logs.cloudwatch.log_stream_name_pattern: >- + type=${type};application={{ .application.id }};scope={{ .scope.id }};deploy={{ .deployment.id }};instance=${instance};container=${container} + nullplatform.logs.cloudwatch.region: us-east-1 + {{- $global := index .k8s_modifiers "global" }} + {{- if $global }} + {{- $annotations := index $global "annotations" }} + {{- if $annotations }} +{{ data.ToYAML $annotations | indent 8 }} + {{- end }} + {{- end }} + {{- $deployment := index .k8s_modifiers "deployment" }} + {{- if $deployment }} + {{- $annotations := index $deployment "annotations" }} + {{- if $annotations }} +{{ data.ToYAML $annotations | indent 8 }} + {{- end }} + {{- end }} + spec: + {{- $deployment := index .k8s_modifiers "deployment" }} + {{- if $deployment }} + {{- $tolerations := index $deployment "tolerations" }} + {{- if $tolerations }} + tolerations: +{{ data.ToYAML $tolerations | indent 6 }} + {{- end }} + {{- $nodeSelector := index $deployment "nodeselector" }} + {{- if $nodeSelector }} + nodeSelector: +{{ data.ToYAML $nodeSelector | indent 8 }} + {{- end }} + {{- end }} + {{- if .pull_secrets.ENABLED }} + imagePullSecrets: + {{- range $secret := .pull_secrets.SECRETS }} + - name: {{ $secret }} + {{- end }} + {{- end }} + {{- if .service_account_name }} + serviceAccountName: {{ .service_account_name }} + {{- end }} + containers: + - name: application + envFrom: + - secretRef: + name: s-{{ .scope.id }}-d-{{ .deployment.id }} + {{- if .parameters.results }} + env: + {{- range .parameters.results }} + {{- if and (eq .type "file") (gt (len .values) 0) }} + {{- $key := .name | strings.ToLower | regexp.Replace "[^a-z0-9]+" "-" | strings.Trim "-" }} + - name: {{ printf "app-data-%s" $key }} + value: {{ .destination_path | quote }} + {{- end }} + {{- end }} + {{- end }} + image: {{ .asset.url }} + resources: + limits: + cpu: {{ .scope.capabilities.cpu_millicores }}m + memory: {{ .scope.capabilities.ram_memory }}Mi + requests: + cpu: {{ .scope.capabilities.cpu_millicores }}m + memory: {{ .scope.capabilities.ram_memory }}Mi + imagePullPolicy: IfNotPresent + volumeMounts: + {{- if .parameters.results }} + {{- range .parameters.results }} + {{- if and (eq .type "file") }} + {{- if gt (len .values) 0 }} + {{- $key := .name | strings.ToLower | regexp.Replace "[^a-z0-9]+" "-" | strings.Trim "-" }} + - name: {{ printf "file-%s" $key }} + mountPath: {{ .destination_path | quote }} + subPath: {{ filepath.Base .destination_path | quote }} + readOnly: true + {{- end }} + {{- end }} + {{- end }} + {{- end }} + volumes: + {{- if .parameters.results }} + {{- range .parameters.results }} + {{- if and (eq .type "file") }} + {{- if gt (len .values) 0 }} + {{- $key := .name | strings.ToLower | regexp.Replace "[^a-z0-9]+" "-" | strings.Trim "-" }} + - name: {{ printf "file-%s" $key }} + secret: + secretName: s-{{ $.scope.id }}-d-{{ $.deployment.id }}-files + items: + - key: {{ printf "app-file-%s" $key }} + path: {{ filepath.Base .destination_path | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + restartPolicy: OnFailure + securityContext: + runAsUser: 0 +{{- else }} schedule: "{{ .scope.capabilities.cron }}" concurrencyPolicy: {{ .scope.capabilities.concurrency_policy }} successfulJobsHistoryLimit: {{ .scope.capabilities.history_limit }} @@ -196,3 +331,4 @@ spec: restartPolicy: OnFailure securityContext: runAsUser: 0 +{{- end }} diff --git a/scheduled_task/deployment/tests/build_deployment.bats b/scheduled_task/deployment/tests/build_deployment.bats index aa29dd2b..8c7a1f3e 100644 --- a/scheduled_task/deployment/tests/build_deployment.bats +++ b/scheduled_task/deployment/tests/build_deployment.bats @@ -164,3 +164,83 @@ JSON local secret_files_file="$OUTPUT_DIR/secret-files-scope-123-deploy-456.yaml" [ ! -f "$secret_files_file" ] || [ ! -s "$secret_files_file" ] } + +# ============================================================================= +# Run-once mode — cron == "run-once" renders a Job instead of a CronJob +# ============================================================================= +@test "build_deployment: scheduled mode renders a CronJob with a schedule" { + unset -f gomplate + + export CONTEXT="$(_render_context)" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + assert_contains "$(cat "$deploy_file")" "kind: CronJob" + assert_contains "$(cat "$deploy_file")" 'schedule: "*/5 * * * *"' + assert_contains "$(cat "$deploy_file")" "jobTemplate:" +} + +@test "build_deployment: run-once mode renders a Job that runs on deploy" { + unset -f gomplate + + # Default JOB_WAIT_TIMEOUT (600) drives the Job's activeDeadlineSeconds. + export CONTEXT="$(_render_context | jq '.scope.capabilities.cron = "run-once"')" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + local rendered="$(cat "$deploy_file")" + + # A one-shot Job, not a CronJob — none of the CronJob-only fields appear. + assert_contains "$rendered" "kind: Job" + ! grep -q "kind: CronJob" "$deploy_file" + ! grep -q "schedule:" "$deploy_file" + ! grep -q "concurrencyPolicy:" "$deploy_file" + ! grep -q "jobTemplate:" "$deploy_file" + + # Job-only knobs: retries -> backoffLimit, timeout -> activeDeadlineSeconds, + # plus a TTL so finished one-shot Jobs are garbage-collected. + assert_contains "$rendered" "backoffLimit: 0" + assert_contains "$rendered" "activeDeadlineSeconds: 600" + assert_contains "$rendered" "ttlSecondsAfterFinished: 86400" + + # The pod block is intact one level shallower (spec.template.spec). + assert_contains "$rendered" "name: job-scope-123-deploy-456" + assert_contains "$rendered" "- name: application" + assert_contains "$rendered" "name: s-scope-123-d-deploy-456" + assert_contains "$rendered" "restartPolicy: OnFailure" +} + +@test "build_deployment: run-once Job keeps file-parameter volumes and mounts" { + unset -f gomplate + + export CONTEXT="$(_render_context | jq '.scope.capabilities.cron = "run-once"')" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + local rendered="$(cat "$deploy_file")" + + assert_contains "$rendered" "- name: app-data-api-p12-cert" + assert_contains "$rendered" 'value: "/app-data/[2026-05-27] cert.p12"' + assert_contains "$rendered" 'mountPath: "/app-data/[2026-05-27] cert.p12"' + assert_contains "$rendered" "secretName: s-scope-123-d-deploy-456-files" + assert_contains "$rendered" "key: app-file-api-p12-cert" +} + +@test "build_deployment: JOB_WAIT_TIMEOUT overrides the Job activeDeadlineSeconds" { + unset -f gomplate + + export JOB_WAIT_TIMEOUT=120 + export CONTEXT="$(_render_context | jq '.scope.capabilities.cron = "run-once"')" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + assert_contains "$(cat "$deploy_file")" "activeDeadlineSeconds: 120" +} diff --git a/scheduled_task/deployment/tests/wait_job.bats b/scheduled_task/deployment/tests/wait_job.bats new file mode 100644 index 00000000..233f1757 --- /dev/null +++ b/scheduled_task/deployment/tests/wait_job.bats @@ -0,0 +1,200 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for scheduled_task/deployment/wait_job - wait for a run-once Job. +# +# Contract: +# - Scheduled (CronJob) deployments have nothing to complete: the script is a +# no-op and exits 0 without touching kubectl. +# - Run-once deployments wait for the Job named job-$SCOPE_ID-$DEPLOYMENT_ID to +# reach Complete (exit 0) or Failed/timeout (exit 1, with actionable logs). +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + + # The workflow loads `log` via a `load logging` step; the script assumes it + # exists. Mock it here (errors go to stderr, like the real one). + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + # No real waiting in tests. + sleep() { :; } + export -f sleep + + export K8S_NAMESPACE="default-namespace" + export JOB_WAIT_TIMEOUT=30 + + # Base CONTEXT: a run-once, deployed scope. + export CONTEXT='{ + "scope": { + "id": "scope-123", + "capabilities": { "cron": "run-once" } + }, + "deployment": { "id": "deploy-456" }, + "providers": { + "container-orchestration": { + "cluster": { "namespace": "provider-namespace" } + } + } + }' + + # Default kubectl mock: the Job is Complete. + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{"conditions":[{"type":"Complete","status":"True"}]}}' + return 0 + ;; + *) + return 0 + ;; + esac + } + export -f kubectl +} + +teardown() { + unset -f kubectl log sleep +} + +# ============================================================================= +# No-op for scheduled (CronJob) deployments +# ============================================================================= +@test "wait_job: scheduled mode is a no-op and never calls kubectl" { + export CONTEXT=$(echo "$CONTEXT" | jq '.scope.capabilities.cron = "*/5 * * * *"') + + # Any kubectl call would be a bug in this path. + kubectl() { echo "kubectl must not be called: $*" >&2; return 1; } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "🔍 Not a run-once deployment (cron='*/5 * * * *'), nothing to wait for" +} + +# ============================================================================= +# Success: Job completes +# ============================================================================= +@test "wait_job: run-once success - waits for the Job to complete" { + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "📝 Waiting for job job-scope-123-deploy-456 to complete (timeout 30s)" + assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" +} + +# ============================================================================= +# Failure: Job fails +# ============================================================================= +@test "wait_job: run-once failure - exits 1 with the Job failure reason" { + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{"conditions":[{"type":"Failed","status":"True","message":"BackoffLimitExceeded"}]}}' + return 0 + ;; + *) return 0 ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ Job job-scope-123-deploy-456 failed: BackoffLimitExceeded" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Inspect the failed run from the logs screen" +} + +# ============================================================================= +# Timeout: Job never reaches a terminal condition +# ============================================================================= +@test "wait_job: run-once timeout - exits 1 when the Job never completes" { + export JOB_WAIT_TIMEOUT=5 # -> MAX_ITERATIONS = 1 + + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{}}' + return 0 + ;; + *) return 0 ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ Timeout waiting for job job-scope-123-deploy-456 to complete after 5s" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Raise JOB_WAIT_TIMEOUT in the scope configuration if the task legitimately needs more time" +} + +# ============================================================================= +# Namespace resolution / set -u guard +# ============================================================================= +@test "wait_job: resolves the namespace from the provider" { + # kubectl only answers for the provider namespace; any other namespace fails. + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{"conditions":[{"type":"Complete","status":"True"}]}}' + return 0 + ;; + *) echo "unexpected namespace: $*" >&2; return 1 ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" +} + +@test "wait_job: does not abort under set -u when K8S_NAMESPACE is unset" { + unset K8S_NAMESPACE + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" +} + +# ============================================================================= +# Workflow wiring — the deploy workflows must run wait_job in place of the +# base "wait deployment active" step, on both the initial and blue-green paths. +# A missing wiring here silently drops the run-once completion wait. +# ============================================================================= +@test "initial workflow replaces 'wait deployment active' with wait_job after apply" { + run python3 - "$BATS_TEST_DIRNAME/../workflows/initial.yaml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +apply = next(s for s in wf["steps"] if s.get("name") == "apply") +post = apply["post"] +assert post["name"] == "wait deployment active", post +assert post["action"] == "replace", post +assert post["file"] == "$OVERRIDES_PATH/deployment/wait_job", post +print("ok") +PY + [ "$status" -eq 0 ] + assert_contains "$output" "ok" +} + +@test "blue_green workflow replaces 'wait deployment active' with wait_job after apply" { + run python3 - "$BATS_TEST_DIRNAME/../workflows/blue_green.yaml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +apply = next(s for s in wf["steps"] if s.get("name") == "apply") +post = apply["post"] +assert post["name"] == "wait deployment active", post +assert post["action"] == "replace", post +assert post["file"] == "$OVERRIDES_PATH/deployment/wait_job", post +print("ok") +PY + [ "$status" -eq 0 ] + assert_contains "$output" "ok" +} diff --git a/scheduled_task/deployment/wait_job b/scheduled_task/deployment/wait_job new file mode 100755 index 00000000..3c3ee1fc --- /dev/null +++ b/scheduled_task/deployment/wait_job @@ -0,0 +1,69 @@ +#!/bin/bash + +set -euo pipefail + +# Only "run-once" deployments have a Job to wait on. Scheduled (CronJob) +# deployments have nothing to complete, so this is a no-op for them — which is +# what keeps the workflow step static across both modes. +CRON=$(echo "$CONTEXT" | jq -r '.scope.capabilities.cron // empty') + +if [[ "$CRON" != "run-once" ]]; then + log debug "🔍 Not a run-once deployment (cron='$CRON'), nothing to wait for" + exit 0 +fi + +K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "${K8S_NAMESPACE:-nullplatform}" ' + .providers["container-orchestration"].cluster.namespace // $default +') + +SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') +DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.deployment.id') +JOB="job-$SCOPE_ID-$DEPLOYMENT_ID" + +TIMEOUT="${JOB_WAIT_TIMEOUT:-600}" +INTERVAL=5 +# The Job's own activeDeadlineSeconds equals TIMEOUT, so poll a little past it: +# that way Kubernetes marks the Job Failed (DeadlineExceeded) first and we report +# that specific reason instead of this generic watchdog message. +GRACE=15 +MAX_ITERATIONS=$(( (TIMEOUT + GRACE) / INTERVAL )) +[ "$MAX_ITERATIONS" -lt 1 ] && MAX_ITERATIONS=1 + +log info "📝 Waiting for job $JOB to complete (timeout ${TIMEOUT}s)" + +for (( iteration=1; iteration<=MAX_ITERATIONS; iteration++ )); do + STATUS=$(kubectl get job "$JOB" -n "$K8S_NAMESPACE" -o json 2>/dev/null || echo "") + + if [[ -n "$STATUS" ]]; then + COMPLETE=$(echo "$STATUS" | jq -r '[.status.conditions[]? | select(.type=="Complete" and .status=="True")] | length') + FAILED=$(echo "$STATUS" | jq -r '[.status.conditions[]? | select(.type=="Failed" and .status=="True")] | length') + + if [[ "$COMPLETE" -gt 0 ]]; then + log info "✅ Job $JOB completed successfully" + exit 0 + fi + + if [[ "$FAILED" -gt 0 ]]; then + REASON=$(echo "$STATUS" | jq -r '[.status.conditions[]? | select(.type=="Failed" and .status=="True") | .message] | first // "see the logs screen for details"') + log error "❌ Job $JOB failed: $REASON" + log error "" + log error "🔧 How to fix:" + log error " • Inspect the failed run from the logs screen" + log error "" + exit 1 + fi + fi + + sleep "$INTERVAL" +done + +log error "❌ Timeout waiting for job $JOB to complete after ${TIMEOUT}s" +log error "" +log error "💡 Possible causes:" +log error " The task is taking longer than the configured timeout, or it is stuck" +log error "" +log error "🔧 How to fix:" +log error " • Inspect the run from the logs screen" +log error " • Raise JOB_WAIT_TIMEOUT in the scope configuration if the task legitimately needs more time" +log error "" +exit 1 diff --git a/scheduled_task/deployment/workflows/blue_green.yaml b/scheduled_task/deployment/workflows/blue_green.yaml index 78f1d7fe..d623d8a2 100644 --- a/scheduled_task/deployment/workflows/blue_green.yaml +++ b/scheduled_task/deployment/workflows/blue_green.yaml @@ -27,4 +27,6 @@ steps: DRY_RUN: false post: name: wait deployment active - action: skip + action: replace + type: script + file: "$OVERRIDES_PATH/deployment/wait_job" diff --git a/scheduled_task/deployment/workflows/initial.yaml b/scheduled_task/deployment/workflows/initial.yaml index 91a87bde..1387bbd7 100644 --- a/scheduled_task/deployment/workflows/initial.yaml +++ b/scheduled_task/deployment/workflows/initial.yaml @@ -24,4 +24,6 @@ steps: DRY_RUN: false post: name: wait deployment active - action: skip + action: replace + type: script + file: "$OVERRIDES_PATH/deployment/wait_job" diff --git a/scheduled_task/scope/tests/trigger.bats b/scheduled_task/scope/tests/trigger.bats index 18ff2732..30923383 100644 --- a/scheduled_task/scope/tests/trigger.bats +++ b/scheduled_task/scope/tests/trigger.bats @@ -98,13 +98,74 @@ teardown() { } # ============================================================================= -# Error: deployed but no CronJob found +# Error: deployed but neither a CronJob nor a previous Job exists # ============================================================================= -@test "trigger: fails with a clear message when no CronJob exists for the scope" { +@test "trigger: fails with a clear message when no CronJob or previous Job exists" { + # No CronJob and no Job for the scope. + kubectl() { echo ""; return 0; } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ No CronJob or previous Job found for scope 'scope-123' in namespace 'provider-namespace'" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Verify a job exists: kubectl get job -n provider-namespace -l scope_id=scope-123" +} + +# ============================================================================= +# Run-once: no CronJob, clone the most recent Job into a fresh one +# ============================================================================= +@test "trigger: run-once re-runs by cloning the last Job of the scope" { + export CREATED_MANIFEST="$(mktemp)" + kubectl() { case "$*" in "get cronjob -n provider-namespace -l scope_id=scope-123 -o jsonpath={.items[0].metadata.name}") - echo "" + echo "" # run-once scope: no CronJob + return 0 + ;; + "get job -n provider-namespace -l scope_id=scope-123 --sort-by=.metadata.creationTimestamp -o jsonpath={.items[-1:].metadata.name}") + echo "job-scope-123-old" + return 0 + ;; + "get job job-scope-123-old -n provider-namespace -o json") + cat <<'JSON' +{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "job-scope-123-old", + "namespace": "provider-namespace", + "uid": "abc-uid", + "resourceVersion": "12345", + "creationTimestamp": "2026-01-01T00:00:00Z", + "labels": { + "scope_id": "scope-123", + "controller-uid": "abc-uid", + "batch.kubernetes.io/controller-uid": "abc-uid", + "job-name": "job-scope-123-old" + } + }, + "spec": { + "backoffLimit": 0, + "selector": { "matchLabels": { "controller-uid": "abc-uid" } }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { "scope_id": "scope-123", "controller-uid": "abc-uid", "job-name": "job-scope-123-old" } + }, + "spec": { "containers": [ { "name": "application", "image": "x" } ], "restartPolicy": "OnFailure" } + } + }, + "status": { "succeeded": 1 } +} +JSON + return 0 + ;; + "create -n provider-namespace -f -") + cat > "$CREATED_MANIFEST" return 0 ;; *) @@ -116,10 +177,20 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../trigger" - [ "$status" -eq 1 ] - assert_contains "$output" "❌ No CronJob found for scope 'scope-123' in namespace 'provider-namespace'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "🔧 How to fix:" + [ "$status" -eq 0 ] + assert_contains "$output" "📝 Re-running job job-scope-123-old as job-scope-123-1700000000" + assert_contains "$output" "✅ The job job-scope-123-1700000000 was triggered, you can follow the execution from the logs screen" + + # The cloned manifest carries the new name and is stripped of the fields the + # API would reject on create. + local manifest="$(cat "$CREATED_MANIFEST")" + assert_contains "$manifest" "job-scope-123-1700000000" + ! grep -q "controller-uid" "$CREATED_MANIFEST" + ! grep -q "job-scope-123-old" "$CREATED_MANIFEST" + ! grep -q "selector" "$CREATED_MANIFEST" + ! grep -q '"status"' "$CREATED_MANIFEST" + + rm -f "$CREATED_MANIFEST" } # ============================================================================= diff --git a/scheduled_task/scope/trigger b/scheduled_task/scope/trigger index 0aeefc5a..cfa80adb 100644 --- a/scheduled_task/scope/trigger +++ b/scheduled_task/scope/trigger @@ -22,23 +22,48 @@ K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "${K8S_NAMESPACE:-nullplat SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') -JOB=$(kubectl get cronjob -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "") +# Scheduled scopes have a CronJob to spawn a Job from. Run-once scopes don't — +# they leave behind the last Job of the scope, which we clone to re-run. +CRONJOB=$(kubectl get cronjob -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "") -if [[ -z "$JOB" ]]; then - log error "❌ No CronJob found for scope '$SCOPE_ID' in namespace '$K8S_NAMESPACE'" +if [[ -n "$CRONJOB" ]]; then + log info "📝 Triggering job $CRONJOB" + kubectl create job --from="cronjob/$CRONJOB" "$CRONJOB-$(date +%s)" -n "$K8S_NAMESPACE" + log info "✅ The job $CRONJOB was triggered, you can follow the execution from the logs screen" + exit 0 +fi + +# Run-once: clone the most recent Job of the scope into a fresh one. Sorting by +# creationTimestamp and taking the last item gives the latest run. +LAST_JOB=$(kubectl get job -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1:].metadata.name}" 2>/dev/null || echo "") + +if [[ -z "$LAST_JOB" ]]; then + log error "❌ No CronJob or previous Job found for scope '$SCOPE_ID' in namespace '$K8S_NAMESPACE'" log error "" log error "💡 Possible causes:" - log error " The scope's scheduled job may not have been created yet, or the deployment is still in progress" + log error " The scope's job may not have been created yet, or the deployment is still in progress" log error "" log error "🔧 How to fix:" - log error " • Verify the CronJob exists: kubectl get cronjob -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" - log error " • Redeploy the scope if the CronJob is missing" + log error " • Verify a job exists: kubectl get job -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" + log error " • Redeploy the scope if no job is present" log error "" exit 1 fi -log info "📝 Triggering job $JOB" +NEW_JOB="job-$SCOPE_ID-$(date +%s)" + +log info "📝 Re-running job $LAST_JOB as $NEW_JOB" -kubectl create job --from="cronjob/$JOB" "$JOB-$(date +%s)" -n "$K8S_NAMESPACE" +# Clone the Job: drop server-managed fields and the controller-generated +# selector/labels, otherwise the API rejects the create. +kubectl get job "$LAST_JOB" -n "$K8S_NAMESPACE" -o json | jq \ + --arg name "$NEW_JOB" \ + --argjson controller_labels '["controller-uid", "batch.kubernetes.io/controller-uid", "job-name", "batch.kubernetes.io/job-name"]' ' + .metadata |= {name: $name, namespace: .namespace, labels: (.labels // {})} + | del(.metadata.labels[$controller_labels[]], .spec.template.metadata.labels[$controller_labels[]]) + | del(.status) + | del(.spec.selector) + | del(.spec.template.metadata.creationTimestamp) + ' | kubectl create -n "$K8S_NAMESPACE" -f - -log info "✅ The job $JOB was triggered, you can follow the execution from the logs screen" +log info "✅ The job $NEW_JOB was triggered, you can follow the execution from the logs screen" diff --git a/scheduled_task/specs/service-spec.json.tpl b/scheduled_task/specs/service-spec.json.tpl index bcca7590..169d2829 100644 --- a/scheduled_task/specs/service-spec.json.tpl +++ b/scheduled_task/specs/service-spec.json.tpl @@ -133,11 +133,15 @@ "const": "0 0 * * *", "title": "Every day (midnight)" }, + { + "const": "run-once", + "title": "Run once" + }, { "type": "string" } ], - "description": "Specify how often the task should run. You can select a predefined option or enter a standard cron expression for custom schedules.", + "description": "Specify how often the task should run. Pick a predefined schedule, enter a standard cron expression, or choose \"Run once\" to run the task a single time when the scope is deployed.", "title": "Task Frequency", "type": "string" }, @@ -241,6 +245,15 @@ "elements": [ { "label": "Concurrency policy", + "rule": { + "condition": { + "schema": { + "const": "run-once" + }, + "scope": "#/properties/cron" + }, + "effect": "HIDE" + }, "scope": "#/properties/concurrency_policy", "type": "Control" }, @@ -251,6 +264,15 @@ }, { "label": "History", + "rule": { + "condition": { + "schema": { + "const": "run-once" + }, + "scope": "#/properties/cron" + }, + "effect": "HIDE" + }, "scope": "#/properties/history_limit", "type": "Control" } diff --git a/scheduled_task/values.yaml b/scheduled_task/values.yaml index 109d378e..b4f5a95c 100644 --- a/scheduled_task/values.yaml +++ b/scheduled_task/values.yaml @@ -1,2 +1,5 @@ configuration: DEPLOYMENT_TEMPLATE: "$OVERRIDES_PATH/deployment/templates/deployment.yaml.tpl" + # Max seconds a "run-once" deployment waits for its Job to complete before + # marking the deployment failed. Also set as the Job's activeDeadlineSeconds. + JOB_WAIT_TIMEOUT: "600"