From 9c73ebe51d21123802a64f8f6c6452f3069a20e8 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Thu, 23 Jul 2026 16:58:03 -0300 Subject: [PATCH 1/2] feat(scheduled-task): add run-once job mode that runs on deploy --- CHANGELOG.md | 1 + scheduled_task/deployment/build_deployment | 10 +- .../deployment/templates/deployment.yaml.tpl | 138 ++++++++++++++- .../deployment/tests/build_deployment.bats | 80 +++++++++ scheduled_task/deployment/tests/wait_job.bats | 165 ++++++++++++++++++ scheduled_task/deployment/wait_job | 69 ++++++++ .../deployment/workflows/blue_green.yaml | 12 ++ .../deployment/workflows/initial.yaml | 12 ++ scheduled_task/scope/tests/trigger.bats | 85 ++++++++- scheduled_task/scope/trigger | 43 ++++- scheduled_task/specs/service-spec.json.tpl | 6 +- scheduled_task/values.yaml | 3 + 12 files changed, 605 insertions(+), 19 deletions(-) create mode 100644 scheduled_task/deployment/tests/wait_job.bats create mode 100755 scheduled_task/deployment/wait_job diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a860a9..8464e9ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Add a "Run once" option to scheduled task scopes: the task runs a single time when the scope is deployed instead of on a recurring schedule, and the deployment waits for it to finish - k8s scope deployments now report launched and healthy instance counts, so the deployment page shows live "X/Y launched" and "X/Y healthy" progress - Fix: triggering a scheduled task job on a scope that is not deployed now fails with a clear "deploy the scope first" message instead of an opaque error - Add "Kill instance" action to scheduled task scopes to terminate an individual running job instance diff --git a/scheduled_task/deployment/build_deployment b/scheduled_task/deployment/build_deployment index c64f5780..31b9b557 100644 --- a/scheduled_task/deployment/build_deployment +++ b/scheduled_task/deployment/build_deployment @@ -5,7 +5,15 @@ SECRET_PATH="$OUTPUT_DIR/secret-$SCOPE_ID-$DEPLOYMENT_ID.yaml" SECRET_FILES_PATH="$OUTPUT_DIR/secret-files-$SCOPE_ID-$DEPLOYMENT_ID.yaml" CONTEXT_PATH="$OUTPUT_DIR/context-$SCOPE_ID.json" -echo "$CONTEXT" | jq --arg replicas "$REPLICAS" '. + {replicas: $replicas}' > "$CONTEXT_PATH" +# In "run-once" mode the template renders a Job with activeDeadlineSeconds set to +# this value, so the Job self-terminates at the same deadline the deployment wait +# gives up at. Ignored by the CronJob branch. +JOB_WAIT_TIMEOUT="${JOB_WAIT_TIMEOUT:-600}" + +echo "$CONTEXT" | jq \ + --arg replicas "$REPLICAS" \ + --argjson job_wait_timeout "$JOB_WAIT_TIMEOUT" \ + '. + {replicas: $replicas, job_wait_timeout: $job_wait_timeout}' > "$CONTEXT_PATH" echo "Building Template: $DEPLOYMENT_TEMPLATE to $DEPLOYMENT_PATH" diff --git a/scheduled_task/deployment/templates/deployment.yaml.tpl b/scheduled_task/deployment/templates/deployment.yaml.tpl index c556acfd..b086ffee 100644 --- a/scheduled_task/deployment/templates/deployment.yaml.tpl +++ b/scheduled_task/deployment/templates/deployment.yaml.tpl @@ -1,5 +1,5 @@ apiVersion: batch/v1 -kind: CronJob +kind: {{ if eq .scope.capabilities.cron "run-once" }}Job{{ else }}CronJob{{ end }} metadata: name: job-{{ .scope.id }}-{{ .deployment.id }} namespace: {{ .k8s_namespace }} @@ -31,6 +31,141 @@ metadata: {{- end }} {{- end }} spec: +{{- if eq .scope.capabilities.cron "run-once" }} + backoffLimit: {{ .scope.capabilities.retries }} + activeDeadlineSeconds: {{ .job_wait_timeout }} + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + name: d-{{ .scope.id }}-{{ .deployment.id }} + app.kubernetes.io/part-of: {{ .component }} + nullplatform: "true" + account: "{{ .account.slug }}" + account_id: "{{ .account.id }}" + namespace: "{{ .namespace.slug }}" + namespace_id: "{{ .namespace.id }}" + application: "{{ .application.slug }}" + application_id: "{{ .application.id }}" + scope: "{{ .scope.slug }}" + scope_id: "{{ .scope.id }}" + deployment_id: "{{ .deployment.id }}" + {{- $global := index .k8s_modifiers "global" }} + {{- if $global }} + {{- $labels := index $global "labels" }} + {{- if $labels }} +{{ data.ToYAML $labels | indent 8 }} + {{- end }} + {{- end }} + {{- $deployment := index .k8s_modifiers "deployment" }} + {{- if $deployment }} + {{- $labels := index $deployment "labels" }} + {{- if $labels }} +{{ data.ToYAML $labels | indent 8 }} + {{- end }} + {{- end }} + annotations: + nullplatform.logs.cloudwatch: 'true' + nullplatform.logs.cloudwatch.log_group_name: {{ .namespace.slug }}.{{ .application.slug }} + nullplatform.logs.cloudwatch.log_stream_log_retention_days: '7' + nullplatform.logs.cloudwatch.log_stream_name_pattern: >- + type=${type};application={{ .application.id }};scope={{ .scope.id }};deploy={{ .deployment.id }};instance=${instance};container=${container} + nullplatform.logs.cloudwatch.region: us-east-1 + {{- $global := index .k8s_modifiers "global" }} + {{- if $global }} + {{- $annotations := index $global "annotations" }} + {{- if $annotations }} +{{ data.ToYAML $annotations | indent 8 }} + {{- end }} + {{- end }} + {{- $deployment := index .k8s_modifiers "deployment" }} + {{- if $deployment }} + {{- $annotations := index $deployment "annotations" }} + {{- if $annotations }} +{{ data.ToYAML $annotations | indent 8 }} + {{- end }} + {{- end }} + spec: + {{- $deployment := index .k8s_modifiers "deployment" }} + {{- if $deployment }} + {{- $tolerations := index $deployment "tolerations" }} + {{- if $tolerations }} + tolerations: +{{ data.ToYAML $tolerations | indent 6 }} + {{- end }} + {{- $nodeSelector := index $deployment "nodeselector" }} + {{- if $nodeSelector }} + nodeSelector: +{{ data.ToYAML $nodeSelector | indent 8 }} + {{- end }} + {{- end }} + {{- if .pull_secrets.ENABLED }} + imagePullSecrets: + {{- range $secret := .pull_secrets.SECRETS }} + - name: {{ $secret }} + {{- end }} + {{- end }} + {{- if .service_account_name }} + serviceAccountName: {{ .service_account_name }} + {{- end }} + containers: + - name: application + envFrom: + - secretRef: + name: s-{{ .scope.id }}-d-{{ .deployment.id }} + {{- if .parameters.results }} + env: + {{- range .parameters.results }} + {{- if and (eq .type "file") (gt (len .values) 0) }} + {{- $key := .name | strings.ToLower | regexp.Replace "[^a-z0-9]+" "-" | strings.Trim "-" }} + - name: {{ printf "app-data-%s" $key }} + value: {{ .destination_path | quote }} + {{- end }} + {{- end }} + {{- end }} + image: {{ .asset.url }} + resources: + limits: + cpu: {{ .scope.capabilities.cpu_millicores }}m + memory: {{ .scope.capabilities.ram_memory }}Mi + requests: + cpu: {{ .scope.capabilities.cpu_millicores }}m + memory: {{ .scope.capabilities.ram_memory }}Mi + imagePullPolicy: IfNotPresent + volumeMounts: + {{- if .parameters.results }} + {{- range .parameters.results }} + {{- if and (eq .type "file") }} + {{- if gt (len .values) 0 }} + {{- $key := .name | strings.ToLower | regexp.Replace "[^a-z0-9]+" "-" | strings.Trim "-" }} + - name: {{ printf "file-%s" $key }} + mountPath: {{ .destination_path | quote }} + subPath: {{ filepath.Base .destination_path | quote }} + readOnly: true + {{- end }} + {{- end }} + {{- end }} + {{- end }} + volumes: + {{- if .parameters.results }} + {{- range .parameters.results }} + {{- if and (eq .type "file") }} + {{- if gt (len .values) 0 }} + {{- $key := .name | strings.ToLower | regexp.Replace "[^a-z0-9]+" "-" | strings.Trim "-" }} + - name: {{ printf "file-%s" $key }} + secret: + secretName: s-{{ $.scope.id }}-d-{{ $.deployment.id }}-files + items: + - key: {{ printf "app-file-%s" $key }} + path: {{ filepath.Base .destination_path | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + restartPolicy: OnFailure + securityContext: + runAsUser: 0 +{{- else }} schedule: "{{ .scope.capabilities.cron }}" concurrencyPolicy: {{ .scope.capabilities.concurrency_policy }} successfulJobsHistoryLimit: {{ .scope.capabilities.history_limit }} @@ -196,3 +331,4 @@ spec: restartPolicy: OnFailure securityContext: runAsUser: 0 +{{- end }} diff --git a/scheduled_task/deployment/tests/build_deployment.bats b/scheduled_task/deployment/tests/build_deployment.bats index aa29dd2b..8c7a1f3e 100644 --- a/scheduled_task/deployment/tests/build_deployment.bats +++ b/scheduled_task/deployment/tests/build_deployment.bats @@ -164,3 +164,83 @@ JSON local secret_files_file="$OUTPUT_DIR/secret-files-scope-123-deploy-456.yaml" [ ! -f "$secret_files_file" ] || [ ! -s "$secret_files_file" ] } + +# ============================================================================= +# Run-once mode — cron == "run-once" renders a Job instead of a CronJob +# ============================================================================= +@test "build_deployment: scheduled mode renders a CronJob with a schedule" { + unset -f gomplate + + export CONTEXT="$(_render_context)" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + assert_contains "$(cat "$deploy_file")" "kind: CronJob" + assert_contains "$(cat "$deploy_file")" 'schedule: "*/5 * * * *"' + assert_contains "$(cat "$deploy_file")" "jobTemplate:" +} + +@test "build_deployment: run-once mode renders a Job that runs on deploy" { + unset -f gomplate + + # Default JOB_WAIT_TIMEOUT (600) drives the Job's activeDeadlineSeconds. + export CONTEXT="$(_render_context | jq '.scope.capabilities.cron = "run-once"')" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + local rendered="$(cat "$deploy_file")" + + # A one-shot Job, not a CronJob — none of the CronJob-only fields appear. + assert_contains "$rendered" "kind: Job" + ! grep -q "kind: CronJob" "$deploy_file" + ! grep -q "schedule:" "$deploy_file" + ! grep -q "concurrencyPolicy:" "$deploy_file" + ! grep -q "jobTemplate:" "$deploy_file" + + # Job-only knobs: retries -> backoffLimit, timeout -> activeDeadlineSeconds, + # plus a TTL so finished one-shot Jobs are garbage-collected. + assert_contains "$rendered" "backoffLimit: 0" + assert_contains "$rendered" "activeDeadlineSeconds: 600" + assert_contains "$rendered" "ttlSecondsAfterFinished: 86400" + + # The pod block is intact one level shallower (spec.template.spec). + assert_contains "$rendered" "name: job-scope-123-deploy-456" + assert_contains "$rendered" "- name: application" + assert_contains "$rendered" "name: s-scope-123-d-deploy-456" + assert_contains "$rendered" "restartPolicy: OnFailure" +} + +@test "build_deployment: run-once Job keeps file-parameter volumes and mounts" { + unset -f gomplate + + export CONTEXT="$(_render_context | jq '.scope.capabilities.cron = "run-once"')" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + local rendered="$(cat "$deploy_file")" + + assert_contains "$rendered" "- name: app-data-api-p12-cert" + assert_contains "$rendered" 'value: "/app-data/[2026-05-27] cert.p12"' + assert_contains "$rendered" 'mountPath: "/app-data/[2026-05-27] cert.p12"' + assert_contains "$rendered" "secretName: s-scope-123-d-deploy-456-files" + assert_contains "$rendered" "key: app-file-api-p12-cert" +} + +@test "build_deployment: JOB_WAIT_TIMEOUT overrides the Job activeDeadlineSeconds" { + unset -f gomplate + + export JOB_WAIT_TIMEOUT=120 + export CONTEXT="$(_render_context | jq '.scope.capabilities.cron = "run-once"')" + + run bash "$BATS_TEST_DIRNAME/../build_deployment" + [ "$status" -eq 0 ] + + local deploy_file="$OUTPUT_DIR/deployment-scope-123-deploy-456.yaml" + assert_contains "$(cat "$deploy_file")" "activeDeadlineSeconds: 120" +} diff --git a/scheduled_task/deployment/tests/wait_job.bats b/scheduled_task/deployment/tests/wait_job.bats new file mode 100644 index 00000000..196c9862 --- /dev/null +++ b/scheduled_task/deployment/tests/wait_job.bats @@ -0,0 +1,165 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for scheduled_task/deployment/wait_job - wait for a run-once Job. +# +# Contract: +# - Scheduled (CronJob) deployments have nothing to complete: the script is a +# no-op and exits 0 without touching kubectl. +# - Run-once deployments wait for the Job named job-$SCOPE_ID-$DEPLOYMENT_ID to +# reach Complete (exit 0) or Failed/timeout (exit 1, with actionable logs). +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + + # The workflow loads `log` via a `load logging` step; the script assumes it + # exists. Mock it here (errors go to stderr, like the real one). + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + # No real waiting in tests. + sleep() { :; } + export -f sleep + + export K8S_NAMESPACE="default-namespace" + export JOB_WAIT_TIMEOUT=30 + + # Base CONTEXT: a run-once, deployed scope. + export CONTEXT='{ + "scope": { + "id": "scope-123", + "capabilities": { "cron": "run-once" } + }, + "deployment": { "id": "deploy-456" }, + "providers": { + "container-orchestration": { + "cluster": { "namespace": "provider-namespace" } + } + } + }' + + # Default kubectl mock: the Job is Complete. + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{"conditions":[{"type":"Complete","status":"True"}]}}' + return 0 + ;; + *) + return 0 + ;; + esac + } + export -f kubectl +} + +teardown() { + unset -f kubectl log sleep +} + +# ============================================================================= +# No-op for scheduled (CronJob) deployments +# ============================================================================= +@test "wait_job: scheduled mode is a no-op and never calls kubectl" { + export CONTEXT=$(echo "$CONTEXT" | jq '.scope.capabilities.cron = "*/5 * * * *"') + + # Any kubectl call would be a bug in this path. + kubectl() { echo "kubectl must not be called: $*" >&2; return 1; } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "🔍 Not a run-once deployment (cron='*/5 * * * *'), nothing to wait for" +} + +# ============================================================================= +# Success: Job completes +# ============================================================================= +@test "wait_job: run-once success - waits for the Job to complete" { + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "📝 Waiting for job job-scope-123-deploy-456 to complete (timeout 30s)" + assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" +} + +# ============================================================================= +# Failure: Job fails +# ============================================================================= +@test "wait_job: run-once failure - exits 1 with the Job failure reason" { + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{"conditions":[{"type":"Failed","status":"True","message":"BackoffLimitExceeded"}]}}' + return 0 + ;; + *) return 0 ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ Job job-scope-123-deploy-456 failed: BackoffLimitExceeded" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Inspect the failed run from the logs screen" +} + +# ============================================================================= +# Timeout: Job never reaches a terminal condition +# ============================================================================= +@test "wait_job: run-once timeout - exits 1 when the Job never completes" { + export JOB_WAIT_TIMEOUT=5 # -> MAX_ITERATIONS = 1 + + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{}}' + return 0 + ;; + *) return 0 ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ Timeout waiting for job job-scope-123-deploy-456 to complete after 5s" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Raise JOB_WAIT_TIMEOUT in the scope configuration if the task legitimately needs more time" +} + +# ============================================================================= +# Namespace resolution / set -u guard +# ============================================================================= +@test "wait_job: resolves the namespace from the provider" { + # kubectl only answers for the provider namespace; any other namespace fails. + kubectl() { + case "$*" in + "get job job-scope-123-deploy-456 -n provider-namespace -o json") + echo '{"status":{"conditions":[{"type":"Complete","status":"True"}]}}' + return 0 + ;; + *) echo "unexpected namespace: $*" >&2; return 1 ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" +} + +@test "wait_job: does not abort under set -u when K8S_NAMESPACE is unset" { + unset K8S_NAMESPACE + + run bash "$BATS_TEST_DIRNAME/../wait_job" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" +} diff --git a/scheduled_task/deployment/wait_job b/scheduled_task/deployment/wait_job new file mode 100755 index 00000000..76030c18 --- /dev/null +++ b/scheduled_task/deployment/wait_job @@ -0,0 +1,69 @@ +#!/bin/bash + +set -euo pipefail + +# Only "run-once" deployments have a Job to wait on. Scheduled (CronJob) +# deployments have nothing to complete, so this is a no-op for them — which is +# what keeps the workflow step static across both modes. +CRON=$(echo "$CONTEXT" | jq -r '.scope.capabilities.cron // empty') + +if [[ "$CRON" != "run-once" ]]; then + log debug "🔍 Not a run-once deployment (cron='$CRON'), nothing to wait for" + exit 0 +fi + +K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "${K8S_NAMESPACE:-nullplatform}" ' + .providers["container-orchestration"].cluster.namespace // $default +') + +SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') +DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.deployment.id') +JOB="job-$SCOPE_ID-$DEPLOYMENT_ID" + +TIMEOUT="${JOB_WAIT_TIMEOUT:-600}" +INTERVAL=5 +MAX_ITERATIONS=$(( TIMEOUT / INTERVAL )) +[ "$MAX_ITERATIONS" -lt 1 ] && MAX_ITERATIONS=1 + +log info "📝 Waiting for job $JOB to complete (timeout ${TIMEOUT}s)" + +iteration=0 +while true; do + ((++iteration)) + if [ "$iteration" -gt "$MAX_ITERATIONS" ]; then + log error "❌ Timeout waiting for job $JOB to complete after ${TIMEOUT}s" + log error "" + log error "💡 Possible causes:" + log error " The task is taking longer than the configured timeout, or it is stuck" + log error "" + log error "🔧 How to fix:" + log error " • Inspect the run from the logs screen" + log error " • Raise JOB_WAIT_TIMEOUT in the scope configuration if the task legitimately needs more time" + log error "" + exit 1 + fi + + STATUS=$(kubectl get job "$JOB" -n "$K8S_NAMESPACE" -o json 2>/dev/null || echo "") + + if [[ -n "$STATUS" ]]; then + COMPLETE=$(echo "$STATUS" | jq -r '[.status.conditions[]? | select(.type=="Complete" and .status=="True")] | length') + FAILED=$(echo "$STATUS" | jq -r '[.status.conditions[]? | select(.type=="Failed" and .status=="True")] | length') + + if [[ "$COMPLETE" -gt 0 ]]; then + log info "✅ Job $JOB completed successfully" + exit 0 + fi + + if [[ "$FAILED" -gt 0 ]]; then + REASON=$(echo "$STATUS" | jq -r '[.status.conditions[]? | select(.type=="Failed" and .status=="True") | .message] | first // "see the logs screen for details"') + log error "❌ Job $JOB failed: $REASON" + log error "" + log error "🔧 How to fix:" + log error " • Inspect the failed run from the logs screen" + log error "" + exit 1 + fi + fi + + sleep "$INTERVAL" +done diff --git a/scheduled_task/deployment/workflows/blue_green.yaml b/scheduled_task/deployment/workflows/blue_green.yaml index 78f1d7fe..d60fd245 100644 --- a/scheduled_task/deployment/workflows/blue_green.yaml +++ b/scheduled_task/deployment/workflows/blue_green.yaml @@ -1,6 +1,15 @@ include: - "$SERVICE_PATH/values.yaml" steps: + - name: load logging + type: script + file: "$OVERRIDES_PATH/logging" + output: + - name: log + type: function + parameters: + level: string + message: string - name: route traffic action: skip - name: create deployment @@ -28,3 +37,6 @@ steps: post: name: wait deployment active action: skip + - name: wait for job + type: script + file: "$OVERRIDES_PATH/deployment/wait_job" diff --git a/scheduled_task/deployment/workflows/initial.yaml b/scheduled_task/deployment/workflows/initial.yaml index 91a87bde..1986ae7e 100644 --- a/scheduled_task/deployment/workflows/initial.yaml +++ b/scheduled_task/deployment/workflows/initial.yaml @@ -1,6 +1,15 @@ include: - "$SERVICE_PATH/values.yaml" steps: + - name: load logging + type: script + file: "$OVERRIDES_PATH/logging" + output: + - name: log + type: function + parameters: + level: string + message: string - name: validate alb target group capacity action: skip - name: route traffic @@ -25,3 +34,6 @@ steps: post: name: wait deployment active action: skip + - name: wait for job + type: script + file: "$OVERRIDES_PATH/deployment/wait_job" diff --git a/scheduled_task/scope/tests/trigger.bats b/scheduled_task/scope/tests/trigger.bats index 18ff2732..30923383 100644 --- a/scheduled_task/scope/tests/trigger.bats +++ b/scheduled_task/scope/tests/trigger.bats @@ -98,13 +98,74 @@ teardown() { } # ============================================================================= -# Error: deployed but no CronJob found +# Error: deployed but neither a CronJob nor a previous Job exists # ============================================================================= -@test "trigger: fails with a clear message when no CronJob exists for the scope" { +@test "trigger: fails with a clear message when no CronJob or previous Job exists" { + # No CronJob and no Job for the scope. + kubectl() { echo ""; return 0; } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ No CronJob or previous Job found for scope 'scope-123' in namespace 'provider-namespace'" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Verify a job exists: kubectl get job -n provider-namespace -l scope_id=scope-123" +} + +# ============================================================================= +# Run-once: no CronJob, clone the most recent Job into a fresh one +# ============================================================================= +@test "trigger: run-once re-runs by cloning the last Job of the scope" { + export CREATED_MANIFEST="$(mktemp)" + kubectl() { case "$*" in "get cronjob -n provider-namespace -l scope_id=scope-123 -o jsonpath={.items[0].metadata.name}") - echo "" + echo "" # run-once scope: no CronJob + return 0 + ;; + "get job -n provider-namespace -l scope_id=scope-123 --sort-by=.metadata.creationTimestamp -o jsonpath={.items[-1:].metadata.name}") + echo "job-scope-123-old" + return 0 + ;; + "get job job-scope-123-old -n provider-namespace -o json") + cat <<'JSON' +{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "job-scope-123-old", + "namespace": "provider-namespace", + "uid": "abc-uid", + "resourceVersion": "12345", + "creationTimestamp": "2026-01-01T00:00:00Z", + "labels": { + "scope_id": "scope-123", + "controller-uid": "abc-uid", + "batch.kubernetes.io/controller-uid": "abc-uid", + "job-name": "job-scope-123-old" + } + }, + "spec": { + "backoffLimit": 0, + "selector": { "matchLabels": { "controller-uid": "abc-uid" } }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { "scope_id": "scope-123", "controller-uid": "abc-uid", "job-name": "job-scope-123-old" } + }, + "spec": { "containers": [ { "name": "application", "image": "x" } ], "restartPolicy": "OnFailure" } + } + }, + "status": { "succeeded": 1 } +} +JSON + return 0 + ;; + "create -n provider-namespace -f -") + cat > "$CREATED_MANIFEST" return 0 ;; *) @@ -116,10 +177,20 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../trigger" - [ "$status" -eq 1 ] - assert_contains "$output" "❌ No CronJob found for scope 'scope-123' in namespace 'provider-namespace'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "🔧 How to fix:" + [ "$status" -eq 0 ] + assert_contains "$output" "📝 Re-running job job-scope-123-old as job-scope-123-1700000000" + assert_contains "$output" "✅ The job job-scope-123-1700000000 was triggered, you can follow the execution from the logs screen" + + # The cloned manifest carries the new name and is stripped of the fields the + # API would reject on create. + local manifest="$(cat "$CREATED_MANIFEST")" + assert_contains "$manifest" "job-scope-123-1700000000" + ! grep -q "controller-uid" "$CREATED_MANIFEST" + ! grep -q "job-scope-123-old" "$CREATED_MANIFEST" + ! grep -q "selector" "$CREATED_MANIFEST" + ! grep -q '"status"' "$CREATED_MANIFEST" + + rm -f "$CREATED_MANIFEST" } # ============================================================================= diff --git a/scheduled_task/scope/trigger b/scheduled_task/scope/trigger index 0aeefc5a..7a1b1356 100644 --- a/scheduled_task/scope/trigger +++ b/scheduled_task/scope/trigger @@ -22,23 +22,48 @@ K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "${K8S_NAMESPACE:-nullplat SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') -JOB=$(kubectl get cronjob -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "") +# Scheduled scopes have a CronJob to spawn a Job from. Run-once scopes don't — +# they leave behind the last Job of the scope, which we clone to re-run. +CRONJOB=$(kubectl get cronjob -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "") -if [[ -z "$JOB" ]]; then - log error "❌ No CronJob found for scope '$SCOPE_ID' in namespace '$K8S_NAMESPACE'" +if [[ -n "$CRONJOB" ]]; then + log info "📝 Triggering job $CRONJOB" + kubectl create job --from="cronjob/$CRONJOB" "$CRONJOB-$(date +%s)" -n "$K8S_NAMESPACE" + log info "✅ The job $CRONJOB was triggered, you can follow the execution from the logs screen" + exit 0 +fi + +# Run-once: clone the most recent Job of the scope into a fresh one. Sorting by +# creationTimestamp and taking the last item gives the latest run. +LAST_JOB=$(kubectl get job -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1:].metadata.name}" 2>/dev/null || echo "") + +if [[ -z "$LAST_JOB" ]]; then + log error "❌ No CronJob or previous Job found for scope '$SCOPE_ID' in namespace '$K8S_NAMESPACE'" log error "" log error "💡 Possible causes:" - log error " The scope's scheduled job may not have been created yet, or the deployment is still in progress" + log error " The scope's job may not have been created yet, or the deployment is still in progress" log error "" log error "🔧 How to fix:" - log error " • Verify the CronJob exists: kubectl get cronjob -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" - log error " • Redeploy the scope if the CronJob is missing" + log error " • Verify a job exists: kubectl get job -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" + log error " • Redeploy the scope if no job is present" log error "" exit 1 fi -log info "📝 Triggering job $JOB" +NEW_JOB="job-$SCOPE_ID-$(date +%s)" + +log info "📝 Re-running job $LAST_JOB as $NEW_JOB" -kubectl create job --from="cronjob/$JOB" "$JOB-$(date +%s)" -n "$K8S_NAMESPACE" +# Clone the Job: drop server-managed fields and the controller-generated +# selector/labels, otherwise the API rejects the create. +kubectl get job "$LAST_JOB" -n "$K8S_NAMESPACE" -o json | jq \ + --arg name "$NEW_JOB" ' + .metadata |= {name: $name, namespace: .namespace, labels: (.labels // {})} + | del(.metadata.labels["controller-uid"], .metadata.labels["batch.kubernetes.io/controller-uid"], .metadata.labels["job-name"], .metadata.labels["batch.kubernetes.io/job-name"]) + | del(.status) + | del(.spec.selector) + | del(.spec.template.metadata.creationTimestamp) + | del(.spec.template.metadata.labels["controller-uid"], .spec.template.metadata.labels["batch.kubernetes.io/controller-uid"], .spec.template.metadata.labels["job-name"], .spec.template.metadata.labels["batch.kubernetes.io/job-name"]) + ' | kubectl create -n "$K8S_NAMESPACE" -f - -log info "✅ The job $JOB was triggered, you can follow the execution from the logs screen" +log info "✅ The job $NEW_JOB was triggered, you can follow the execution from the logs screen" diff --git a/scheduled_task/specs/service-spec.json.tpl b/scheduled_task/specs/service-spec.json.tpl index 4ffc81ac..d86774b3 100644 --- a/scheduled_task/specs/service-spec.json.tpl +++ b/scheduled_task/specs/service-spec.json.tpl @@ -122,11 +122,15 @@ "const": "0 0 * * *", "title": "Every day (midnight)" }, + { + "const": "run-once", + "title": "Run once" + }, { "type": "string" } ], - "description": "Specify how often the task should run. You can select a predefined option or enter a standard cron expression for custom schedules.", + "description": "Specify how often the task should run. Pick a predefined schedule, enter a standard cron expression, or choose \"Run once\" to run the task a single time when the scope is deployed.", "title": "Task Frequency", "type": "string" }, diff --git a/scheduled_task/values.yaml b/scheduled_task/values.yaml index 109d378e..b4f5a95c 100644 --- a/scheduled_task/values.yaml +++ b/scheduled_task/values.yaml @@ -1,2 +1,5 @@ configuration: DEPLOYMENT_TEMPLATE: "$OVERRIDES_PATH/deployment/templates/deployment.yaml.tpl" + # Max seconds a "run-once" deployment waits for its Job to complete before + # marking the deployment failed. Also set as the Job's activeDeadlineSeconds. + JOB_WAIT_TIMEOUT: "600" From b8822ad500396cfdda9ec239349f33f95bb7f754 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Thu, 23 Jul 2026 17:10:00 -0300 Subject: [PATCH 2/2] refactor(scheduled-task): robust run-once wait wiring and UX polish from review --- scheduled_task/deployment/tests/wait_job.bats | 35 +++++++++++++++++++ scheduled_task/deployment/wait_job | 34 +++++++++--------- .../deployment/workflows/blue_green.yaml | 16 ++------- .../deployment/workflows/initial.yaml | 16 ++------- scheduled_task/scope/trigger | 6 ++-- scheduled_task/specs/service-spec.json.tpl | 18 ++++++++++ 6 files changed, 79 insertions(+), 46 deletions(-) diff --git a/scheduled_task/deployment/tests/wait_job.bats b/scheduled_task/deployment/tests/wait_job.bats index 196c9862..233f1757 100644 --- a/scheduled_task/deployment/tests/wait_job.bats +++ b/scheduled_task/deployment/tests/wait_job.bats @@ -163,3 +163,38 @@ teardown() { [ "$status" -eq 0 ] assert_contains "$output" "✅ Job job-scope-123-deploy-456 completed successfully" } + +# ============================================================================= +# Workflow wiring — the deploy workflows must run wait_job in place of the +# base "wait deployment active" step, on both the initial and blue-green paths. +# A missing wiring here silently drops the run-once completion wait. +# ============================================================================= +@test "initial workflow replaces 'wait deployment active' with wait_job after apply" { + run python3 - "$BATS_TEST_DIRNAME/../workflows/initial.yaml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +apply = next(s for s in wf["steps"] if s.get("name") == "apply") +post = apply["post"] +assert post["name"] == "wait deployment active", post +assert post["action"] == "replace", post +assert post["file"] == "$OVERRIDES_PATH/deployment/wait_job", post +print("ok") +PY + [ "$status" -eq 0 ] + assert_contains "$output" "ok" +} + +@test "blue_green workflow replaces 'wait deployment active' with wait_job after apply" { + run python3 - "$BATS_TEST_DIRNAME/../workflows/blue_green.yaml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +apply = next(s for s in wf["steps"] if s.get("name") == "apply") +post = apply["post"] +assert post["name"] == "wait deployment active", post +assert post["action"] == "replace", post +assert post["file"] == "$OVERRIDES_PATH/deployment/wait_job", post +print("ok") +PY + [ "$status" -eq 0 ] + assert_contains "$output" "ok" +} diff --git a/scheduled_task/deployment/wait_job b/scheduled_task/deployment/wait_job index 76030c18..3c3ee1fc 100755 --- a/scheduled_task/deployment/wait_job +++ b/scheduled_task/deployment/wait_job @@ -22,27 +22,16 @@ JOB="job-$SCOPE_ID-$DEPLOYMENT_ID" TIMEOUT="${JOB_WAIT_TIMEOUT:-600}" INTERVAL=5 -MAX_ITERATIONS=$(( TIMEOUT / INTERVAL )) +# The Job's own activeDeadlineSeconds equals TIMEOUT, so poll a little past it: +# that way Kubernetes marks the Job Failed (DeadlineExceeded) first and we report +# that specific reason instead of this generic watchdog message. +GRACE=15 +MAX_ITERATIONS=$(( (TIMEOUT + GRACE) / INTERVAL )) [ "$MAX_ITERATIONS" -lt 1 ] && MAX_ITERATIONS=1 log info "📝 Waiting for job $JOB to complete (timeout ${TIMEOUT}s)" -iteration=0 -while true; do - ((++iteration)) - if [ "$iteration" -gt "$MAX_ITERATIONS" ]; then - log error "❌ Timeout waiting for job $JOB to complete after ${TIMEOUT}s" - log error "" - log error "💡 Possible causes:" - log error " The task is taking longer than the configured timeout, or it is stuck" - log error "" - log error "🔧 How to fix:" - log error " • Inspect the run from the logs screen" - log error " • Raise JOB_WAIT_TIMEOUT in the scope configuration if the task legitimately needs more time" - log error "" - exit 1 - fi - +for (( iteration=1; iteration<=MAX_ITERATIONS; iteration++ )); do STATUS=$(kubectl get job "$JOB" -n "$K8S_NAMESPACE" -o json 2>/dev/null || echo "") if [[ -n "$STATUS" ]]; then @@ -67,3 +56,14 @@ while true; do sleep "$INTERVAL" done + +log error "❌ Timeout waiting for job $JOB to complete after ${TIMEOUT}s" +log error "" +log error "💡 Possible causes:" +log error " The task is taking longer than the configured timeout, or it is stuck" +log error "" +log error "🔧 How to fix:" +log error " • Inspect the run from the logs screen" +log error " • Raise JOB_WAIT_TIMEOUT in the scope configuration if the task legitimately needs more time" +log error "" +exit 1 diff --git a/scheduled_task/deployment/workflows/blue_green.yaml b/scheduled_task/deployment/workflows/blue_green.yaml index d60fd245..d623d8a2 100644 --- a/scheduled_task/deployment/workflows/blue_green.yaml +++ b/scheduled_task/deployment/workflows/blue_green.yaml @@ -1,15 +1,6 @@ include: - "$SERVICE_PATH/values.yaml" steps: - - name: load logging - type: script - file: "$OVERRIDES_PATH/logging" - output: - - name: log - type: function - parameters: - level: string - message: string - name: route traffic action: skip - name: create deployment @@ -36,7 +27,6 @@ steps: DRY_RUN: false post: name: wait deployment active - action: skip - - name: wait for job - type: script - file: "$OVERRIDES_PATH/deployment/wait_job" + action: replace + type: script + file: "$OVERRIDES_PATH/deployment/wait_job" diff --git a/scheduled_task/deployment/workflows/initial.yaml b/scheduled_task/deployment/workflows/initial.yaml index 1986ae7e..1387bbd7 100644 --- a/scheduled_task/deployment/workflows/initial.yaml +++ b/scheduled_task/deployment/workflows/initial.yaml @@ -1,15 +1,6 @@ include: - "$SERVICE_PATH/values.yaml" steps: - - name: load logging - type: script - file: "$OVERRIDES_PATH/logging" - output: - - name: log - type: function - parameters: - level: string - message: string - name: validate alb target group capacity action: skip - name: route traffic @@ -33,7 +24,6 @@ steps: DRY_RUN: false post: name: wait deployment active - action: skip - - name: wait for job - type: script - file: "$OVERRIDES_PATH/deployment/wait_job" + action: replace + type: script + file: "$OVERRIDES_PATH/deployment/wait_job" diff --git a/scheduled_task/scope/trigger b/scheduled_task/scope/trigger index 7a1b1356..cfa80adb 100644 --- a/scheduled_task/scope/trigger +++ b/scheduled_task/scope/trigger @@ -57,13 +57,13 @@ log info "📝 Re-running job $LAST_JOB as $NEW_JOB" # Clone the Job: drop server-managed fields and the controller-generated # selector/labels, otherwise the API rejects the create. kubectl get job "$LAST_JOB" -n "$K8S_NAMESPACE" -o json | jq \ - --arg name "$NEW_JOB" ' + --arg name "$NEW_JOB" \ + --argjson controller_labels '["controller-uid", "batch.kubernetes.io/controller-uid", "job-name", "batch.kubernetes.io/job-name"]' ' .metadata |= {name: $name, namespace: .namespace, labels: (.labels // {})} - | del(.metadata.labels["controller-uid"], .metadata.labels["batch.kubernetes.io/controller-uid"], .metadata.labels["job-name"], .metadata.labels["batch.kubernetes.io/job-name"]) + | del(.metadata.labels[$controller_labels[]], .spec.template.metadata.labels[$controller_labels[]]) | del(.status) | del(.spec.selector) | del(.spec.template.metadata.creationTimestamp) - | del(.spec.template.metadata.labels["controller-uid"], .spec.template.metadata.labels["batch.kubernetes.io/controller-uid"], .spec.template.metadata.labels["job-name"], .spec.template.metadata.labels["batch.kubernetes.io/job-name"]) ' | kubectl create -n "$K8S_NAMESPACE" -f - log info "✅ The job $NEW_JOB was triggered, you can follow the execution from the logs screen" diff --git a/scheduled_task/specs/service-spec.json.tpl b/scheduled_task/specs/service-spec.json.tpl index d86774b3..94b9bf37 100644 --- a/scheduled_task/specs/service-spec.json.tpl +++ b/scheduled_task/specs/service-spec.json.tpl @@ -234,6 +234,15 @@ "elements": [ { "label": "Concurrency policy", + "rule": { + "condition": { + "schema": { + "const": "run-once" + }, + "scope": "#/properties/cron" + }, + "effect": "HIDE" + }, "scope": "#/properties/concurrency_policy", "type": "Control" }, @@ -244,6 +253,15 @@ }, { "label": "History", + "rule": { + "condition": { + "schema": { + "const": "run-once" + }, + "scope": "#/properties/cron" + }, + "effect": "HIDE" + }, "scope": "#/properties/history_limit", "type": "Control" }